...

Source file src/k8s.io/kubernetes/pkg/controller/cronjob/cronjob_controllerv2_test.go

Documentation: k8s.io/kubernetes/pkg/controller/cronjob

     1  /*
     2  Copyright 2020 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package cronjob
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"reflect"
    23  	"sort"
    24  	"strings"
    25  	"testing"
    26  	"time"
    27  
    28  	batchv1 "k8s.io/api/batch/v1"
    29  	v1 "k8s.io/api/core/v1"
    30  	"k8s.io/apimachinery/pkg/api/errors"
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  	"k8s.io/apimachinery/pkg/runtime"
    33  	"k8s.io/apimachinery/pkg/runtime/schema"
    34  	"k8s.io/apimachinery/pkg/types"
    35  	"k8s.io/client-go/informers"
    36  	"k8s.io/client-go/kubernetes/fake"
    37  	"k8s.io/client-go/tools/record"
    38  	"k8s.io/client-go/util/workqueue"
    39  	"k8s.io/klog/v2/ktesting"
    40  	"k8s.io/utils/pointer"
    41  
    42  	_ "k8s.io/kubernetes/pkg/apis/batch/install"
    43  	_ "k8s.io/kubernetes/pkg/apis/core/install"
    44  	"k8s.io/kubernetes/pkg/controller"
    45  )
    46  
    47  var (
    48  	shortDead  int64 = 10
    49  	mediumDead int64 = 2 * 60 * 60
    50  	longDead   int64 = 1000000
    51  	noDead     int64 = -12345
    52  
    53  	errorSchedule = "obvious error schedule"
    54  	// schedule is hourly on the hour
    55  	onTheHour = "0 * * * ?"
    56  	everyHour = "@every 1h"
    57  
    58  	errorTimeZone = "bad timezone"
    59  	newYork       = "America/New_York"
    60  )
    61  
    62  // returns a cronJob with some fields filled in.
    63  func cronJob() batchv1.CronJob {
    64  	return batchv1.CronJob{
    65  		ObjectMeta: metav1.ObjectMeta{
    66  			Name:              "mycronjob",
    67  			Namespace:         "snazzycats",
    68  			UID:               types.UID("1a2b3c"),
    69  			CreationTimestamp: metav1.Time{Time: justBeforeTheHour()},
    70  		},
    71  		Spec: batchv1.CronJobSpec{
    72  			Schedule:          "* * * * ?",
    73  			ConcurrencyPolicy: "Allow",
    74  			JobTemplate: batchv1.JobTemplateSpec{
    75  				ObjectMeta: metav1.ObjectMeta{
    76  					Labels:      map[string]string{"a": "b"},
    77  					Annotations: map[string]string{"x": "y"},
    78  				},
    79  				Spec: jobSpec(),
    80  			},
    81  		},
    82  	}
    83  }
    84  
    85  func jobSpec() batchv1.JobSpec {
    86  	one := int32(1)
    87  	return batchv1.JobSpec{
    88  		Parallelism: &one,
    89  		Completions: &one,
    90  		Template: v1.PodTemplateSpec{
    91  			ObjectMeta: metav1.ObjectMeta{
    92  				Labels: map[string]string{
    93  					"foo": "bar",
    94  				},
    95  			},
    96  			Spec: v1.PodSpec{
    97  				Containers: []v1.Container{
    98  					{Image: "foo/bar"},
    99  				},
   100  			},
   101  		},
   102  	}
   103  }
   104  
   105  func justASecondBeforeTheHour() time.Time {
   106  	T1, err := time.Parse(time.RFC3339, "2016-05-19T09:59:59Z")
   107  	if err != nil {
   108  		panic("test setup error")
   109  	}
   110  	return T1
   111  }
   112  
   113  func justAfterThePriorHour() time.Time {
   114  	T1, err := time.Parse(time.RFC3339, "2016-05-19T09:01:00Z")
   115  	if err != nil {
   116  		panic("test setup error")
   117  	}
   118  	return T1
   119  }
   120  
   121  func justBeforeThePriorHour() time.Time {
   122  	T1, err := time.Parse(time.RFC3339, "2016-05-19T08:59:00Z")
   123  	if err != nil {
   124  		panic("test setup error")
   125  	}
   126  	return T1
   127  }
   128  
   129  func justAfterTheHour() *time.Time {
   130  	T1, err := time.Parse(time.RFC3339, "2016-05-19T10:01:00Z")
   131  	if err != nil {
   132  		panic("test setup error")
   133  	}
   134  	return &T1
   135  }
   136  
   137  func justAfterTheHourInZone(tz string) time.Time {
   138  	location, err := time.LoadLocation(tz)
   139  	if err != nil {
   140  		panic("tz error: " + err.Error())
   141  	}
   142  
   143  	T1, err := time.ParseInLocation(time.RFC3339, "2016-05-19T10:01:00Z", location)
   144  	if err != nil {
   145  		panic("test setup error: " + err.Error())
   146  	}
   147  	return T1
   148  }
   149  
   150  func justBeforeTheHour() time.Time {
   151  	T1, err := time.Parse(time.RFC3339, "2016-05-19T09:59:00Z")
   152  	if err != nil {
   153  		panic("test setup error")
   154  	}
   155  	return T1
   156  }
   157  
   158  func justBeforeTheNextHour() time.Time {
   159  	T1, err := time.Parse(time.RFC3339, "2016-05-19T10:59:00Z")
   160  	if err != nil {
   161  		panic("test setup error")
   162  	}
   163  	return T1
   164  }
   165  
   166  func weekAfterTheHour() time.Time {
   167  	T1, err := time.Parse(time.RFC3339, "2016-05-26T10:00:00Z")
   168  	if err != nil {
   169  		panic("test setup error")
   170  	}
   171  	return T1
   172  }
   173  
   174  func TestControllerV2SyncCronJob(t *testing.T) {
   175  	// Check expectations on deadline parameters
   176  	if shortDead/60/60 >= 1 {
   177  		t.Errorf("shortDead should be less than one hour")
   178  	}
   179  
   180  	if mediumDead/60/60 < 1 || mediumDead/60/60 >= 24 {
   181  		t.Errorf("mediumDead should be between one hour and one day")
   182  	}
   183  
   184  	if longDead/60/60/24 < 10 {
   185  		t.Errorf("longDead should be at least ten days")
   186  	}
   187  
   188  	testCases := map[string]struct {
   189  		// cj spec
   190  		concurrencyPolicy batchv1.ConcurrencyPolicy
   191  		suspend           bool
   192  		schedule          string
   193  		timeZone          *string
   194  		deadline          int64
   195  
   196  		// cj status
   197  		ranPreviously bool
   198  		stillActive   bool
   199  
   200  		// environment
   201  		cronjobCreationTime time.Time
   202  		jobCreationTime     time.Time
   203  		lastScheduleTime    time.Time
   204  		now                 time.Time
   205  		jobCreateError      error
   206  		jobGetErr           error
   207  
   208  		// expectations
   209  		expectCreate               bool
   210  		expectDelete               bool
   211  		expectActive               int
   212  		expectedWarnings           int
   213  		expectErr                  bool
   214  		expectRequeueAfter         bool
   215  		expectedRequeueDuration    time.Duration
   216  		expectUpdateStatus         bool
   217  		jobStillNotFoundInLister   bool
   218  		jobPresentInCJActiveStatus bool
   219  	}{
   220  		"never ran, not valid schedule, A": {
   221  			concurrencyPolicy:          "Allow",
   222  			schedule:                   errorSchedule,
   223  			deadline:                   noDead,
   224  			jobCreationTime:            justAfterThePriorHour(),
   225  			now:                        justBeforeTheHour(),
   226  			expectedWarnings:           1,
   227  			jobPresentInCJActiveStatus: true,
   228  		},
   229  		"never ran, not valid schedule, F": {
   230  			concurrencyPolicy:          "Forbid",
   231  			schedule:                   errorSchedule,
   232  			deadline:                   noDead,
   233  			jobCreationTime:            justAfterThePriorHour(),
   234  			now:                        justBeforeTheHour(),
   235  			expectedWarnings:           1,
   236  			jobPresentInCJActiveStatus: true,
   237  		},
   238  		"never ran, not valid schedule, R": {
   239  			concurrencyPolicy:          "Forbid",
   240  			schedule:                   errorSchedule,
   241  			deadline:                   noDead,
   242  			jobCreationTime:            justAfterThePriorHour(),
   243  			now:                        justBeforeTheHour(),
   244  			expectedWarnings:           1,
   245  			jobPresentInCJActiveStatus: true,
   246  		},
   247  		"never ran, not valid time zone": {
   248  			concurrencyPolicy:          "Allow",
   249  			schedule:                   onTheHour,
   250  			timeZone:                   &errorTimeZone,
   251  			deadline:                   noDead,
   252  			jobCreationTime:            justAfterThePriorHour(),
   253  			now:                        justBeforeTheHour(),
   254  			expectedWarnings:           1,
   255  			jobPresentInCJActiveStatus: true,
   256  		},
   257  		"never ran, not time, A": {
   258  			concurrencyPolicy:          "Allow",
   259  			schedule:                   onTheHour,
   260  			deadline:                   noDead,
   261  			jobCreationTime:            justAfterThePriorHour(),
   262  			now:                        justBeforeTheHour(),
   263  			expectRequeueAfter:         true,
   264  			expectedRequeueDuration:    1*time.Minute + nextScheduleDelta,
   265  			jobPresentInCJActiveStatus: true},
   266  		"never ran, not time, F": {
   267  			concurrencyPolicy:          "Forbid",
   268  			schedule:                   onTheHour,
   269  			deadline:                   noDead,
   270  			jobCreationTime:            justAfterThePriorHour(),
   271  			now:                        justBeforeTheHour(),
   272  			expectRequeueAfter:         true,
   273  			expectedRequeueDuration:    1*time.Minute + nextScheduleDelta,
   274  			jobPresentInCJActiveStatus: true,
   275  		},
   276  		"never ran, not time, R": {
   277  			concurrencyPolicy:          "Replace",
   278  			schedule:                   onTheHour,
   279  			deadline:                   noDead,
   280  			jobCreationTime:            justAfterThePriorHour(),
   281  			now:                        justBeforeTheHour(),
   282  			expectRequeueAfter:         true,
   283  			expectedRequeueDuration:    1*time.Minute + nextScheduleDelta,
   284  			jobPresentInCJActiveStatus: true,
   285  		},
   286  		"never ran, not time in zone": {
   287  			concurrencyPolicy:          "Allow",
   288  			schedule:                   onTheHour,
   289  			timeZone:                   &newYork,
   290  			deadline:                   noDead,
   291  			jobCreationTime:            justAfterThePriorHour(),
   292  			now:                        justBeforeTheHour(),
   293  			expectRequeueAfter:         true,
   294  			expectedRequeueDuration:    1*time.Minute + nextScheduleDelta,
   295  			jobPresentInCJActiveStatus: true,
   296  		},
   297  		"never ran, is time, A": {
   298  			concurrencyPolicy:          "Allow",
   299  			schedule:                   onTheHour,
   300  			deadline:                   noDead,
   301  			jobCreationTime:            justAfterThePriorHour(),
   302  			now:                        *justAfterTheHour(),
   303  			expectCreate:               true,
   304  			expectActive:               1,
   305  			expectRequeueAfter:         true,
   306  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   307  			expectUpdateStatus:         true,
   308  			jobPresentInCJActiveStatus: true,
   309  		},
   310  		"never ran, is time, F": {
   311  			concurrencyPolicy:          "Forbid",
   312  			schedule:                   onTheHour,
   313  			deadline:                   noDead,
   314  			jobCreationTime:            justAfterThePriorHour(),
   315  			now:                        *justAfterTheHour(),
   316  			expectCreate:               true,
   317  			expectActive:               1,
   318  			expectRequeueAfter:         true,
   319  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   320  			expectUpdateStatus:         true,
   321  			jobPresentInCJActiveStatus: true,
   322  		},
   323  		"never ran, is time, R": {
   324  			concurrencyPolicy:          "Replace",
   325  			schedule:                   onTheHour,
   326  			deadline:                   noDead,
   327  			jobCreationTime:            justAfterThePriorHour(),
   328  			now:                        *justAfterTheHour(),
   329  			expectCreate:               true,
   330  			expectActive:               1,
   331  			expectRequeueAfter:         true,
   332  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   333  			expectUpdateStatus:         true,
   334  			jobPresentInCJActiveStatus: true,
   335  		},
   336  		"never ran, is time in zone, but time zone disabled": {
   337  			concurrencyPolicy:          "Allow",
   338  			schedule:                   onTheHour,
   339  			timeZone:                   &newYork,
   340  			deadline:                   noDead,
   341  			jobCreationTime:            justAfterThePriorHour(),
   342  			now:                        justAfterTheHourInZone(newYork),
   343  			expectCreate:               true,
   344  			expectActive:               1,
   345  			expectRequeueAfter:         true,
   346  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   347  			expectUpdateStatus:         true,
   348  			jobPresentInCJActiveStatus: true,
   349  		},
   350  		"never ran, is time in zone": {
   351  			concurrencyPolicy:          "Allow",
   352  			schedule:                   onTheHour,
   353  			timeZone:                   &newYork,
   354  			deadline:                   noDead,
   355  			jobCreationTime:            justAfterThePriorHour(),
   356  			now:                        justAfterTheHourInZone(newYork),
   357  			expectCreate:               true,
   358  			expectActive:               1,
   359  			expectRequeueAfter:         true,
   360  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   361  			expectUpdateStatus:         true,
   362  			jobPresentInCJActiveStatus: true,
   363  		},
   364  		"never ran, is time in zone, but TZ is also set in schedule": {
   365  			concurrencyPolicy:          "Allow",
   366  			schedule:                   "TZ=UTC " + onTheHour,
   367  			timeZone:                   &newYork,
   368  			deadline:                   noDead,
   369  			jobCreationTime:            justAfterThePriorHour(),
   370  			now:                        justAfterTheHourInZone(newYork),
   371  			expectCreate:               true,
   372  			expectedWarnings:           1,
   373  			expectRequeueAfter:         true,
   374  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   375  			expectUpdateStatus:         true,
   376  			jobPresentInCJActiveStatus: true,
   377  		},
   378  		"never ran, is time, suspended": {
   379  			concurrencyPolicy:          "Allow",
   380  			suspend:                    true,
   381  			schedule:                   onTheHour,
   382  			deadline:                   noDead,
   383  			jobCreationTime:            justAfterThePriorHour(),
   384  			now:                        *justAfterTheHour(),
   385  			jobPresentInCJActiveStatus: true,
   386  		},
   387  		"never ran, is time, past deadline": {
   388  			concurrencyPolicy:          "Allow",
   389  			schedule:                   onTheHour,
   390  			deadline:                   shortDead,
   391  			jobCreationTime:            justAfterThePriorHour(),
   392  			now:                        justAfterTheHour().Add(time.Minute * time.Duration(shortDead+1)),
   393  			expectRequeueAfter:         true,
   394  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute - time.Minute*time.Duration(shortDead+1) + nextScheduleDelta,
   395  			jobPresentInCJActiveStatus: true,
   396  		},
   397  		"never ran, is time, not past deadline": {
   398  			concurrencyPolicy:          "Allow",
   399  			schedule:                   onTheHour,
   400  			deadline:                   longDead,
   401  			jobCreationTime:            justAfterThePriorHour(),
   402  			now:                        *justAfterTheHour(),
   403  			expectCreate:               true,
   404  			expectActive:               1,
   405  			expectRequeueAfter:         true,
   406  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   407  			expectUpdateStatus:         true,
   408  			jobPresentInCJActiveStatus: true,
   409  		},
   410  
   411  		"prev ran but done, not time, A": {
   412  			concurrencyPolicy:          "Allow",
   413  			schedule:                   onTheHour,
   414  			deadline:                   noDead,
   415  			ranPreviously:              true,
   416  			jobCreationTime:            justAfterThePriorHour(),
   417  			now:                        justBeforeTheHour(),
   418  			expectRequeueAfter:         true,
   419  			expectedRequeueDuration:    1*time.Minute + nextScheduleDelta,
   420  			expectUpdateStatus:         true,
   421  			jobPresentInCJActiveStatus: true,
   422  		},
   423  		"prev ran but done, not time, F": {
   424  			concurrencyPolicy:          "Forbid",
   425  			schedule:                   onTheHour,
   426  			deadline:                   noDead,
   427  			ranPreviously:              true,
   428  			jobCreationTime:            justAfterThePriorHour(),
   429  			now:                        justBeforeTheHour(),
   430  			expectRequeueAfter:         true,
   431  			expectedRequeueDuration:    1*time.Minute + nextScheduleDelta,
   432  			expectUpdateStatus:         true,
   433  			jobPresentInCJActiveStatus: true,
   434  		},
   435  		"prev ran but done, not time, R": {
   436  			concurrencyPolicy:          "Replace",
   437  			schedule:                   onTheHour,
   438  			deadline:                   noDead,
   439  			ranPreviously:              true,
   440  			jobCreationTime:            justAfterThePriorHour(),
   441  			now:                        justBeforeTheHour(),
   442  			expectRequeueAfter:         true,
   443  			expectedRequeueDuration:    1*time.Minute + nextScheduleDelta,
   444  			expectUpdateStatus:         true,
   445  			jobPresentInCJActiveStatus: true,
   446  		},
   447  		"prev ran but done, is time, A": {
   448  			concurrencyPolicy:          "Allow",
   449  			schedule:                   onTheHour,
   450  			deadline:                   noDead,
   451  			ranPreviously:              true,
   452  			jobCreationTime:            justAfterThePriorHour(),
   453  			now:                        *justAfterTheHour(),
   454  			expectCreate:               true,
   455  			expectActive:               1,
   456  			expectRequeueAfter:         true,
   457  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   458  			expectUpdateStatus:         true,
   459  			jobPresentInCJActiveStatus: true,
   460  		},
   461  		"prev ran but done, is time, create job failed, A": {
   462  			concurrencyPolicy:          "Allow",
   463  			schedule:                   onTheHour,
   464  			deadline:                   noDead,
   465  			ranPreviously:              true,
   466  			jobCreationTime:            justAfterThePriorHour(),
   467  			now:                        *justAfterTheHour(),
   468  			jobCreateError:             errors.NewAlreadyExists(schema.GroupResource{Resource: "job", Group: "batch"}, ""),
   469  			expectErr:                  false,
   470  			expectUpdateStatus:         true,
   471  			jobPresentInCJActiveStatus: true,
   472  		},
   473  		"prev ran but done, is time, job not present in CJ active status, create job failed, A": {
   474  			concurrencyPolicy:          "Allow",
   475  			schedule:                   onTheHour,
   476  			deadline:                   noDead,
   477  			ranPreviously:              true,
   478  			jobCreationTime:            justAfterThePriorHour(),
   479  			now:                        *justAfterTheHour(),
   480  			jobCreateError:             errors.NewAlreadyExists(schema.GroupResource{Resource: "job", Group: "batch"}, ""),
   481  			expectErr:                  false,
   482  			expectUpdateStatus:         true,
   483  			jobPresentInCJActiveStatus: false,
   484  		},
   485  		"prev ran but done, is time, F": {
   486  			concurrencyPolicy:          "Forbid",
   487  			schedule:                   onTheHour,
   488  			deadline:                   noDead,
   489  			ranPreviously:              true,
   490  			jobCreationTime:            justAfterThePriorHour(),
   491  			now:                        *justAfterTheHour(),
   492  			expectCreate:               true,
   493  			expectActive:               1,
   494  			expectRequeueAfter:         true,
   495  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   496  			expectUpdateStatus:         true,
   497  			jobPresentInCJActiveStatus: true,
   498  		},
   499  		"prev ran but done, is time, R": {
   500  			concurrencyPolicy:          "Replace",
   501  			schedule:                   onTheHour,
   502  			deadline:                   noDead,
   503  			ranPreviously:              true,
   504  			jobCreationTime:            justAfterThePriorHour(),
   505  			now:                        *justAfterTheHour(),
   506  			expectCreate:               true,
   507  			expectActive:               1,
   508  			expectRequeueAfter:         true,
   509  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   510  			expectUpdateStatus:         true,
   511  			jobPresentInCJActiveStatus: true,
   512  		},
   513  		"prev ran but done, is time, suspended": {
   514  			concurrencyPolicy:          "Allow",
   515  			suspend:                    true,
   516  			schedule:                   onTheHour,
   517  			deadline:                   noDead,
   518  			ranPreviously:              true,
   519  			jobCreationTime:            justAfterThePriorHour(),
   520  			now:                        *justAfterTheHour(),
   521  			expectUpdateStatus:         true,
   522  			jobPresentInCJActiveStatus: true,
   523  		},
   524  		"prev ran but done, is time, past deadline": {
   525  			concurrencyPolicy:          "Allow",
   526  			schedule:                   onTheHour,
   527  			deadline:                   shortDead,
   528  			ranPreviously:              true,
   529  			jobCreationTime:            justAfterThePriorHour(),
   530  			now:                        *justAfterTheHour(),
   531  			expectRequeueAfter:         true,
   532  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   533  			expectUpdateStatus:         true,
   534  			jobPresentInCJActiveStatus: true,
   535  		},
   536  		"prev ran but done, is time, not past deadline": {
   537  			concurrencyPolicy:          "Allow",
   538  			schedule:                   onTheHour,
   539  			deadline:                   longDead,
   540  			ranPreviously:              true,
   541  			jobCreationTime:            justAfterThePriorHour(),
   542  			now:                        *justAfterTheHour(),
   543  			expectCreate:               true,
   544  			expectActive:               1,
   545  			expectRequeueAfter:         true,
   546  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   547  			expectUpdateStatus:         true,
   548  			jobPresentInCJActiveStatus: true,
   549  		},
   550  
   551  		"still active, not time, A": {
   552  			concurrencyPolicy:          "Allow",
   553  			schedule:                   onTheHour,
   554  			deadline:                   noDead,
   555  			ranPreviously:              true,
   556  			stillActive:                true,
   557  			jobCreationTime:            justAfterThePriorHour(),
   558  			now:                        justBeforeTheHour(),
   559  			expectActive:               1,
   560  			expectRequeueAfter:         true,
   561  			expectedRequeueDuration:    1*time.Minute + nextScheduleDelta,
   562  			jobPresentInCJActiveStatus: true,
   563  		},
   564  		"still active, not time, F": {
   565  			concurrencyPolicy:          "Forbid",
   566  			schedule:                   onTheHour,
   567  			deadline:                   noDead,
   568  			ranPreviously:              true,
   569  			stillActive:                true,
   570  			jobCreationTime:            justAfterThePriorHour(),
   571  			now:                        justBeforeTheHour(),
   572  			expectActive:               1,
   573  			expectRequeueAfter:         true,
   574  			expectedRequeueDuration:    1*time.Minute + nextScheduleDelta,
   575  			jobPresentInCJActiveStatus: true,
   576  		},
   577  		"still active, not time, R": {
   578  			concurrencyPolicy:          "Replace",
   579  			schedule:                   onTheHour,
   580  			deadline:                   noDead,
   581  			ranPreviously:              true,
   582  			stillActive:                true,
   583  			jobCreationTime:            justAfterThePriorHour(),
   584  			now:                        justBeforeTheHour(),
   585  			expectActive:               1,
   586  			expectRequeueAfter:         true,
   587  			expectedRequeueDuration:    1*time.Minute + nextScheduleDelta,
   588  			jobPresentInCJActiveStatus: true,
   589  		},
   590  		"still active, is time, A": {
   591  			concurrencyPolicy:          "Allow",
   592  			schedule:                   onTheHour,
   593  			deadline:                   noDead,
   594  			ranPreviously:              true,
   595  			stillActive:                true,
   596  			jobCreationTime:            justAfterThePriorHour(),
   597  			now:                        *justAfterTheHour(),
   598  			expectCreate:               true,
   599  			expectActive:               2,
   600  			expectRequeueAfter:         true,
   601  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   602  			expectUpdateStatus:         true,
   603  			jobPresentInCJActiveStatus: true,
   604  		},
   605  		"still active, is time, F": {
   606  			concurrencyPolicy:          "Forbid",
   607  			schedule:                   onTheHour,
   608  			deadline:                   noDead,
   609  			ranPreviously:              true,
   610  			stillActive:                true,
   611  			jobCreationTime:            justAfterThePriorHour(),
   612  			now:                        *justAfterTheHour(),
   613  			expectActive:               1,
   614  			expectRequeueAfter:         true,
   615  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   616  			jobPresentInCJActiveStatus: true,
   617  		},
   618  		"still active, is time, R": {
   619  			concurrencyPolicy:          "Replace",
   620  			schedule:                   onTheHour,
   621  			deadline:                   noDead,
   622  			ranPreviously:              true,
   623  			stillActive:                true,
   624  			jobCreationTime:            justAfterThePriorHour(),
   625  			now:                        *justAfterTheHour(),
   626  			expectCreate:               true,
   627  			expectDelete:               true,
   628  			expectActive:               1,
   629  			expectRequeueAfter:         true,
   630  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   631  			expectUpdateStatus:         true,
   632  			jobPresentInCJActiveStatus: true,
   633  		},
   634  		"still active, is time, get job failed, R": {
   635  			concurrencyPolicy:          "Replace",
   636  			schedule:                   onTheHour,
   637  			deadline:                   noDead,
   638  			ranPreviously:              true,
   639  			stillActive:                true,
   640  			jobCreationTime:            justAfterThePriorHour(),
   641  			now:                        *justAfterTheHour(),
   642  			jobGetErr:                  errors.NewBadRequest("request is invalid"),
   643  			expectActive:               1,
   644  			expectedWarnings:           1,
   645  			jobPresentInCJActiveStatus: true,
   646  		},
   647  		"still active, is time, suspended": {
   648  			concurrencyPolicy:          "Allow",
   649  			suspend:                    true,
   650  			schedule:                   onTheHour,
   651  			deadline:                   noDead,
   652  			ranPreviously:              true,
   653  			stillActive:                true,
   654  			jobCreationTime:            justAfterThePriorHour(),
   655  			now:                        *justAfterTheHour(),
   656  			expectActive:               1,
   657  			jobPresentInCJActiveStatus: true,
   658  		},
   659  		"still active, is time, past deadline": {
   660  			concurrencyPolicy:          "Allow",
   661  			schedule:                   onTheHour,
   662  			deadline:                   shortDead,
   663  			ranPreviously:              true,
   664  			stillActive:                true,
   665  			jobCreationTime:            justAfterThePriorHour(),
   666  			now:                        *justAfterTheHour(),
   667  			expectActive:               1,
   668  			expectRequeueAfter:         true,
   669  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   670  			jobPresentInCJActiveStatus: true,
   671  		},
   672  		"still active, is time, not past deadline": {
   673  			concurrencyPolicy:          "Allow",
   674  			schedule:                   onTheHour,
   675  			deadline:                   longDead,
   676  			ranPreviously:              true,
   677  			stillActive:                true,
   678  			jobCreationTime:            justAfterThePriorHour(),
   679  			now:                        *justAfterTheHour(),
   680  			expectCreate:               true,
   681  			expectActive:               2,
   682  			expectRequeueAfter:         true,
   683  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   684  			expectUpdateStatus:         true,
   685  			jobPresentInCJActiveStatus: true,
   686  		},
   687  
   688  		// Controller should fail to schedule these, as there are too many missed starting times
   689  		// and either no deadline or a too long deadline.
   690  		"prev ran but done, long overdue, not past deadline, A": {
   691  			concurrencyPolicy:          "Allow",
   692  			schedule:                   onTheHour,
   693  			deadline:                   longDead,
   694  			ranPreviously:              true,
   695  			jobCreationTime:            justAfterThePriorHour(),
   696  			now:                        weekAfterTheHour(),
   697  			expectCreate:               true,
   698  			expectActive:               1,
   699  			expectedWarnings:           1,
   700  			expectRequeueAfter:         true,
   701  			expectedRequeueDuration:    1*time.Hour + nextScheduleDelta,
   702  			expectUpdateStatus:         true,
   703  			jobPresentInCJActiveStatus: true,
   704  		},
   705  		"prev ran but done, long overdue, not past deadline, R": {
   706  			concurrencyPolicy:          "Replace",
   707  			schedule:                   onTheHour,
   708  			deadline:                   longDead,
   709  			ranPreviously:              true,
   710  			jobCreationTime:            justAfterThePriorHour(),
   711  			now:                        weekAfterTheHour(),
   712  			expectCreate:               true,
   713  			expectActive:               1,
   714  			expectedWarnings:           1,
   715  			expectRequeueAfter:         true,
   716  			expectedRequeueDuration:    1*time.Hour + nextScheduleDelta,
   717  			expectUpdateStatus:         true,
   718  			jobPresentInCJActiveStatus: true,
   719  		},
   720  		"prev ran but done, long overdue, not past deadline, F": {
   721  			concurrencyPolicy:          "Forbid",
   722  			schedule:                   onTheHour,
   723  			deadline:                   longDead,
   724  			ranPreviously:              true,
   725  			jobCreationTime:            justAfterThePriorHour(),
   726  			now:                        weekAfterTheHour(),
   727  			expectCreate:               true,
   728  			expectActive:               1,
   729  			expectedWarnings:           1,
   730  			expectRequeueAfter:         true,
   731  			expectedRequeueDuration:    1*time.Hour + nextScheduleDelta,
   732  			expectUpdateStatus:         true,
   733  			jobPresentInCJActiveStatus: true,
   734  		},
   735  		"prev ran but done, long overdue, no deadline, A": {
   736  			concurrencyPolicy:          "Allow",
   737  			schedule:                   onTheHour,
   738  			deadline:                   noDead,
   739  			ranPreviously:              true,
   740  			jobCreationTime:            justAfterThePriorHour(),
   741  			now:                        weekAfterTheHour(),
   742  			expectCreate:               true,
   743  			expectActive:               1,
   744  			expectedWarnings:           1,
   745  			expectRequeueAfter:         true,
   746  			expectedRequeueDuration:    1*time.Hour + nextScheduleDelta,
   747  			expectUpdateStatus:         true,
   748  			jobPresentInCJActiveStatus: true,
   749  		},
   750  		"prev ran but done, long overdue, no deadline, R": {
   751  			concurrencyPolicy:          "Replace",
   752  			schedule:                   onTheHour,
   753  			deadline:                   noDead,
   754  			ranPreviously:              true,
   755  			jobCreationTime:            justAfterThePriorHour(),
   756  			now:                        weekAfterTheHour(),
   757  			expectCreate:               true,
   758  			expectActive:               1,
   759  			expectedWarnings:           1,
   760  			expectRequeueAfter:         true,
   761  			expectedRequeueDuration:    1*time.Hour + nextScheduleDelta,
   762  			expectUpdateStatus:         true,
   763  			jobPresentInCJActiveStatus: true,
   764  		},
   765  		"prev ran but done, long overdue, no deadline, F": {
   766  			concurrencyPolicy:          "Forbid",
   767  			schedule:                   onTheHour,
   768  			deadline:                   noDead,
   769  			ranPreviously:              true,
   770  			jobCreationTime:            justAfterThePriorHour(),
   771  			now:                        weekAfterTheHour(),
   772  			expectCreate:               true,
   773  			expectActive:               1,
   774  			expectedWarnings:           1,
   775  			expectRequeueAfter:         true,
   776  			expectedRequeueDuration:    1*time.Hour + nextScheduleDelta,
   777  			expectUpdateStatus:         true,
   778  			jobPresentInCJActiveStatus: true,
   779  		},
   780  
   781  		"prev ran but done, long overdue, past medium deadline, A": {
   782  			concurrencyPolicy:          "Allow",
   783  			schedule:                   onTheHour,
   784  			deadline:                   mediumDead,
   785  			ranPreviously:              true,
   786  			jobCreationTime:            justAfterThePriorHour(),
   787  			now:                        weekAfterTheHour(),
   788  			expectCreate:               true,
   789  			expectActive:               1,
   790  			expectRequeueAfter:         true,
   791  			expectedRequeueDuration:    1*time.Hour + nextScheduleDelta,
   792  			expectUpdateStatus:         true,
   793  			jobPresentInCJActiveStatus: true,
   794  		},
   795  		"prev ran but done, long overdue, past short deadline, A": {
   796  			concurrencyPolicy:          "Allow",
   797  			schedule:                   onTheHour,
   798  			deadline:                   shortDead,
   799  			ranPreviously:              true,
   800  			jobCreationTime:            justAfterThePriorHour(),
   801  			now:                        weekAfterTheHour(),
   802  			expectCreate:               true,
   803  			expectActive:               1,
   804  			expectRequeueAfter:         true,
   805  			expectedRequeueDuration:    1*time.Hour + nextScheduleDelta,
   806  			expectUpdateStatus:         true,
   807  			jobPresentInCJActiveStatus: true,
   808  		},
   809  
   810  		"prev ran but done, long overdue, past medium deadline, R": {
   811  			concurrencyPolicy:          "Replace",
   812  			schedule:                   onTheHour,
   813  			deadline:                   mediumDead,
   814  			ranPreviously:              true,
   815  			jobCreationTime:            justAfterThePriorHour(),
   816  			now:                        weekAfterTheHour(),
   817  			expectCreate:               true,
   818  			expectActive:               1,
   819  			expectRequeueAfter:         true,
   820  			expectedRequeueDuration:    1*time.Hour + nextScheduleDelta,
   821  			expectUpdateStatus:         true,
   822  			jobPresentInCJActiveStatus: true,
   823  		},
   824  		"prev ran but done, long overdue, past short deadline, R": {
   825  			concurrencyPolicy:          "Replace",
   826  			schedule:                   onTheHour,
   827  			deadline:                   shortDead,
   828  			ranPreviously:              true,
   829  			jobCreationTime:            justAfterThePriorHour(),
   830  			now:                        weekAfterTheHour(),
   831  			expectCreate:               true,
   832  			expectActive:               1,
   833  			expectRequeueAfter:         true,
   834  			expectedRequeueDuration:    1*time.Hour + nextScheduleDelta,
   835  			expectUpdateStatus:         true,
   836  			jobPresentInCJActiveStatus: true,
   837  		},
   838  
   839  		"prev ran but done, long overdue, past medium deadline, F": {
   840  			concurrencyPolicy:          "Forbid",
   841  			schedule:                   onTheHour,
   842  			deadline:                   mediumDead,
   843  			ranPreviously:              true,
   844  			jobCreationTime:            justAfterThePriorHour(),
   845  			now:                        weekAfterTheHour(),
   846  			expectCreate:               true,
   847  			expectActive:               1,
   848  			expectRequeueAfter:         true,
   849  			expectedRequeueDuration:    1*time.Hour + nextScheduleDelta,
   850  			expectUpdateStatus:         true,
   851  			jobPresentInCJActiveStatus: true,
   852  		},
   853  		"prev ran but done, long overdue, past short deadline, F": {
   854  			concurrencyPolicy:          "Forbid",
   855  			schedule:                   onTheHour,
   856  			deadline:                   shortDead,
   857  			ranPreviously:              true,
   858  			jobCreationTime:            justAfterThePriorHour(),
   859  			now:                        weekAfterTheHour(),
   860  			expectCreate:               true,
   861  			expectActive:               1,
   862  			expectRequeueAfter:         true,
   863  			expectedRequeueDuration:    1*time.Hour + nextScheduleDelta,
   864  			expectUpdateStatus:         true,
   865  			jobPresentInCJActiveStatus: true,
   866  		},
   867  
   868  		// Tests for time skews
   869  		// the controller sees job is created, takes no actions
   870  		"this ran but done, time drifted back, F": {
   871  			concurrencyPolicy:       "Forbid",
   872  			schedule:                onTheHour,
   873  			deadline:                noDead,
   874  			ranPreviously:           true,
   875  			jobCreationTime:         *justAfterTheHour(),
   876  			now:                     justBeforeTheHour(),
   877  			jobCreateError:          errors.NewAlreadyExists(schema.GroupResource{Resource: "jobs", Group: "batch"}, ""),
   878  			expectRequeueAfter:      true,
   879  			expectedRequeueDuration: 1*time.Minute + nextScheduleDelta,
   880  			expectUpdateStatus:      true,
   881  		},
   882  
   883  		// Tests for slow job lister
   884  		"this started but went missing, not past deadline, A": {
   885  			concurrencyPolicy:          "Allow",
   886  			schedule:                   onTheHour,
   887  			deadline:                   longDead,
   888  			ranPreviously:              true,
   889  			stillActive:                true,
   890  			jobCreationTime:            topOfTheHour().Add(time.Millisecond * 100),
   891  			now:                        justAfterTheHour().Add(time.Millisecond * 100),
   892  			expectActive:               1,
   893  			expectRequeueAfter:         true,
   894  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute - time.Millisecond*100 + nextScheduleDelta,
   895  			jobStillNotFoundInLister:   true,
   896  			jobPresentInCJActiveStatus: true,
   897  		},
   898  		"this started but went missing, not past deadline, f": {
   899  			concurrencyPolicy:          "Forbid",
   900  			schedule:                   onTheHour,
   901  			deadline:                   longDead,
   902  			ranPreviously:              true,
   903  			stillActive:                true,
   904  			jobCreationTime:            topOfTheHour().Add(time.Millisecond * 100),
   905  			now:                        justAfterTheHour().Add(time.Millisecond * 100),
   906  			expectActive:               1,
   907  			expectRequeueAfter:         true,
   908  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute - time.Millisecond*100 + nextScheduleDelta,
   909  			jobStillNotFoundInLister:   true,
   910  			jobPresentInCJActiveStatus: true,
   911  		},
   912  		"this started but went missing, not past deadline, R": {
   913  			concurrencyPolicy:          "Replace",
   914  			schedule:                   onTheHour,
   915  			deadline:                   longDead,
   916  			ranPreviously:              true,
   917  			stillActive:                true,
   918  			jobCreationTime:            topOfTheHour().Add(time.Millisecond * 100),
   919  			now:                        justAfterTheHour().Add(time.Millisecond * 100),
   920  			expectActive:               1,
   921  			expectRequeueAfter:         true,
   922  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute - time.Millisecond*100 + nextScheduleDelta,
   923  			jobStillNotFoundInLister:   true,
   924  			jobPresentInCJActiveStatus: true,
   925  		},
   926  
   927  		// Tests for slow cronjob list
   928  		"this started but is not present in cronjob active list, not past deadline, A": {
   929  			concurrencyPolicy:       "Allow",
   930  			schedule:                onTheHour,
   931  			deadline:                longDead,
   932  			ranPreviously:           true,
   933  			stillActive:             true,
   934  			jobCreationTime:         topOfTheHour().Add(time.Millisecond * 100),
   935  			now:                     justAfterTheHour().Add(time.Millisecond * 100),
   936  			expectActive:            1,
   937  			expectRequeueAfter:      true,
   938  			expectedRequeueDuration: 1*time.Hour - 1*time.Minute - time.Millisecond*100 + nextScheduleDelta,
   939  		},
   940  		"this started but is not present in cronjob active list, not past deadline, f": {
   941  			concurrencyPolicy:       "Forbid",
   942  			schedule:                onTheHour,
   943  			deadline:                longDead,
   944  			ranPreviously:           true,
   945  			stillActive:             true,
   946  			jobCreationTime:         topOfTheHour().Add(time.Millisecond * 100),
   947  			now:                     justAfterTheHour().Add(time.Millisecond * 100),
   948  			expectActive:            1,
   949  			expectRequeueAfter:      true,
   950  			expectedRequeueDuration: 1*time.Hour - 1*time.Minute - time.Millisecond*100 + nextScheduleDelta,
   951  		},
   952  		"this started but is not present in cronjob active list, not past deadline, R": {
   953  			concurrencyPolicy:       "Replace",
   954  			schedule:                onTheHour,
   955  			deadline:                longDead,
   956  			ranPreviously:           true,
   957  			stillActive:             true,
   958  			jobCreationTime:         topOfTheHour().Add(time.Millisecond * 100),
   959  			now:                     justAfterTheHour().Add(time.Millisecond * 100),
   960  			expectActive:            1,
   961  			expectRequeueAfter:      true,
   962  			expectedRequeueDuration: 1*time.Hour - 1*time.Minute - time.Millisecond*100 + nextScheduleDelta,
   963  		},
   964  
   965  		// Tests for @every-style schedule
   966  		"with @every schedule, never ran, not time": {
   967  			concurrencyPolicy:          "Allow",
   968  			schedule:                   everyHour,
   969  			deadline:                   noDead,
   970  			cronjobCreationTime:        justBeforeTheHour(),
   971  			jobCreationTime:            justBeforeTheHour(),
   972  			now:                        *topOfTheHour(),
   973  			expectRequeueAfter:         true,
   974  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
   975  			jobPresentInCJActiveStatus: true,
   976  		},
   977  		"with @every schedule, never ran, is time": {
   978  			concurrencyPolicy:          "Allow",
   979  			schedule:                   everyHour,
   980  			deadline:                   noDead,
   981  			cronjobCreationTime:        justBeforeThePriorHour(),
   982  			jobCreationTime:            justBeforeThePriorHour(),
   983  			now:                        justBeforeTheHour(),
   984  			expectRequeueAfter:         true,
   985  			expectedRequeueDuration:    1*time.Hour + nextScheduleDelta,
   986  			jobPresentInCJActiveStatus: true,
   987  			expectCreate:               true,
   988  			expectActive:               1,
   989  			expectUpdateStatus:         true,
   990  		},
   991  		"with @every schedule, never ran, is time, past deadline": {
   992  			concurrencyPolicy:          "Allow",
   993  			schedule:                   everyHour,
   994  			deadline:                   shortDead,
   995  			cronjobCreationTime:        justBeforeThePriorHour(),
   996  			jobCreationTime:            justBeforeThePriorHour(),
   997  			now:                        justBeforeTheHour().Add(time.Second * time.Duration(shortDead+1)),
   998  			expectRequeueAfter:         true,
   999  			expectedRequeueDuration:    1*time.Hour - time.Second*time.Duration(shortDead+1) + nextScheduleDelta,
  1000  			jobPresentInCJActiveStatus: true,
  1001  		},
  1002  		"with @every schedule, never ran, is time, not past deadline": {
  1003  			concurrencyPolicy:          "Allow",
  1004  			schedule:                   everyHour,
  1005  			deadline:                   longDead,
  1006  			cronjobCreationTime:        justBeforeThePriorHour(),
  1007  			jobCreationTime:            justBeforeThePriorHour(),
  1008  			now:                        justBeforeTheHour().Add(time.Second * time.Duration(shortDead-1)),
  1009  			expectCreate:               true,
  1010  			expectActive:               1,
  1011  			expectRequeueAfter:         true,
  1012  			expectedRequeueDuration:    1*time.Hour - time.Second*time.Duration(shortDead-1) + nextScheduleDelta,
  1013  			expectUpdateStatus:         true,
  1014  			jobPresentInCJActiveStatus: true,
  1015  		},
  1016  		"with @every schedule, prev ran but done, not time": {
  1017  			concurrencyPolicy:          "Allow",
  1018  			schedule:                   everyHour,
  1019  			deadline:                   noDead,
  1020  			ranPreviously:              true,
  1021  			cronjobCreationTime:        justBeforeThePriorHour(),
  1022  			jobCreationTime:            justBeforeThePriorHour(),
  1023  			lastScheduleTime:           justBeforeTheHour(),
  1024  			now:                        *topOfTheHour(),
  1025  			expectRequeueAfter:         true,
  1026  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
  1027  			expectUpdateStatus:         true,
  1028  			jobPresentInCJActiveStatus: true,
  1029  		},
  1030  		"with @every schedule, prev ran but done, is time": {
  1031  			concurrencyPolicy:          "Allow",
  1032  			schedule:                   everyHour,
  1033  			deadline:                   noDead,
  1034  			ranPreviously:              true,
  1035  			cronjobCreationTime:        justBeforeThePriorHour(),
  1036  			jobCreationTime:            justBeforeThePriorHour(),
  1037  			lastScheduleTime:           justBeforeTheHour(),
  1038  			now:                        topOfTheHour().Add(1 * time.Hour),
  1039  			expectCreate:               true,
  1040  			expectActive:               1,
  1041  			expectRequeueAfter:         true,
  1042  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
  1043  			expectUpdateStatus:         true,
  1044  			jobPresentInCJActiveStatus: true,
  1045  		},
  1046  		"with @every schedule, prev ran but done, is time, past deadline": {
  1047  			concurrencyPolicy:          "Allow",
  1048  			schedule:                   everyHour,
  1049  			deadline:                   shortDead,
  1050  			ranPreviously:              true,
  1051  			cronjobCreationTime:        justBeforeThePriorHour(),
  1052  			jobCreationTime:            justBeforeThePriorHour(),
  1053  			lastScheduleTime:           justBeforeTheHour(),
  1054  			now:                        justBeforeTheNextHour().Add(time.Second * time.Duration(shortDead+1)),
  1055  			expectRequeueAfter:         true,
  1056  			expectedRequeueDuration:    1*time.Hour - time.Second*time.Duration(shortDead+1) + nextScheduleDelta,
  1057  			expectUpdateStatus:         true,
  1058  			jobPresentInCJActiveStatus: true,
  1059  		},
  1060  		// This test will fail: the logic around StartingDeadlineSecond in getNextScheduleTime messes up
  1061  		// the time that calculating schedule.Next(earliestTime) is based on. While this works perfectly
  1062  		// well for classic cron scheduled, with @every X, schedule.Next(earliestTime) just returns the time
  1063  		// offset by X relative to the earliestTime.
  1064  		// "with @every schedule, prev ran but done, is time, not past deadline": {
  1065  		// 	concurrencyPolicy:          "Allow",
  1066  		// 	schedule:                   everyHour,
  1067  		// 	deadline:                   shortDead,
  1068  		// 	ranPreviously:              true,
  1069  		// 	cronjobCreationTime:        justBeforeThePriorHour(),
  1070  		// 	jobCreationTime:            justBeforeThePriorHour(),
  1071  		// 	lastScheduleTime:           justBeforeTheHour(),
  1072  		// 	now:                        justBeforeTheNextHour().Add(time.Second * time.Duration(shortDead-1)),
  1073  		// 	expectCreate:               true,
  1074  		// 	expectActive:               1,
  1075  		// 	expectRequeueAfter:         true,
  1076  		// 	expectedRequeueDuration:    1*time.Hour - time.Second*time.Duration(shortDead-1) + nextScheduleDelta,
  1077  		// 	expectUpdateStatus:         true,
  1078  		// 	jobPresentInCJActiveStatus: true,
  1079  		// },
  1080  		"with @every schedule, still active, not time": {
  1081  			concurrencyPolicy:          "Allow",
  1082  			schedule:                   everyHour,
  1083  			deadline:                   noDead,
  1084  			ranPreviously:              true,
  1085  			stillActive:                true,
  1086  			cronjobCreationTime:        justBeforeThePriorHour(),
  1087  			jobCreationTime:            justBeforeTheHour(),
  1088  			lastScheduleTime:           justBeforeTheHour(),
  1089  			now:                        *topOfTheHour(),
  1090  			expectActive:               1,
  1091  			expectRequeueAfter:         true,
  1092  			expectedRequeueDuration:    1*time.Hour - 1*time.Minute + nextScheduleDelta,
  1093  			jobPresentInCJActiveStatus: true,
  1094  		},
  1095  		"with @every schedule, still active, is time": {
  1096  			concurrencyPolicy:          "Allow",
  1097  			schedule:                   everyHour,
  1098  			deadline:                   noDead,
  1099  			ranPreviously:              true,
  1100  			stillActive:                true,
  1101  			cronjobCreationTime:        justBeforeThePriorHour(),
  1102  			jobCreationTime:            justBeforeThePriorHour(),
  1103  			lastScheduleTime:           justBeforeThePriorHour(),
  1104  			now:                        *justAfterTheHour(),
  1105  			expectCreate:               true,
  1106  			expectActive:               2,
  1107  			expectRequeueAfter:         true,
  1108  			expectedRequeueDuration:    1*time.Hour - 2*time.Minute + nextScheduleDelta,
  1109  			expectUpdateStatus:         true,
  1110  			jobPresentInCJActiveStatus: true,
  1111  		},
  1112  		"with @every schedule, still active, is time, past deadline": {
  1113  			concurrencyPolicy:          "Allow",
  1114  			schedule:                   everyHour,
  1115  			deadline:                   shortDead,
  1116  			ranPreviously:              true,
  1117  			stillActive:                true,
  1118  			cronjobCreationTime:        justBeforeThePriorHour(),
  1119  			jobCreationTime:            justBeforeTheHour(),
  1120  			lastScheduleTime:           justBeforeTheHour(),
  1121  			now:                        justBeforeTheNextHour().Add(time.Second * time.Duration(shortDead+1)),
  1122  			expectActive:               1,
  1123  			expectRequeueAfter:         true,
  1124  			expectedRequeueDuration:    1*time.Hour - time.Second*time.Duration(shortDead+1) + nextScheduleDelta,
  1125  			jobPresentInCJActiveStatus: true,
  1126  		},
  1127  		"with @every schedule, still active, is time, not past deadline": {
  1128  			concurrencyPolicy:          "Allow",
  1129  			schedule:                   everyHour,
  1130  			deadline:                   longDead,
  1131  			ranPreviously:              true,
  1132  			stillActive:                true,
  1133  			cronjobCreationTime:        justBeforeThePriorHour(),
  1134  			jobCreationTime:            justBeforeTheHour(),
  1135  			lastScheduleTime:           justBeforeTheHour(),
  1136  			now:                        justBeforeTheNextHour().Add(time.Second * time.Duration(shortDead-1)),
  1137  			expectCreate:               true,
  1138  			expectActive:               2,
  1139  			expectRequeueAfter:         true,
  1140  			expectedRequeueDuration:    1*time.Hour - time.Second*time.Duration(shortDead-1) + nextScheduleDelta,
  1141  			expectUpdateStatus:         true,
  1142  			jobPresentInCJActiveStatus: true,
  1143  		},
  1144  		"with @every schedule, prev ran but done, long overdue, no deadline": {
  1145  			concurrencyPolicy:          "Allow",
  1146  			schedule:                   everyHour,
  1147  			deadline:                   noDead,
  1148  			ranPreviously:              true,
  1149  			cronjobCreationTime:        justAfterThePriorHour(),
  1150  			lastScheduleTime:           *justAfterTheHour(),
  1151  			jobCreationTime:            justAfterThePriorHour(),
  1152  			now:                        weekAfterTheHour(),
  1153  			expectCreate:               true,
  1154  			expectActive:               1,
  1155  			expectedWarnings:           1,
  1156  			expectRequeueAfter:         true,
  1157  			expectedRequeueDuration:    1*time.Minute + nextScheduleDelta,
  1158  			expectUpdateStatus:         true,
  1159  			jobPresentInCJActiveStatus: true,
  1160  		},
  1161  		"with @every schedule, prev ran but done, long overdue, past deadline": {
  1162  			concurrencyPolicy:          "Allow",
  1163  			schedule:                   everyHour,
  1164  			deadline:                   shortDead,
  1165  			ranPreviously:              true,
  1166  			cronjobCreationTime:        justAfterThePriorHour(),
  1167  			lastScheduleTime:           *justAfterTheHour(),
  1168  			jobCreationTime:            justAfterThePriorHour(),
  1169  			now:                        weekAfterTheHour().Add(1 * time.Minute).Add(time.Second * time.Duration(shortDead+1)),
  1170  			expectActive:               1,
  1171  			expectRequeueAfter:         true,
  1172  			expectedRequeueDuration:    1*time.Hour - time.Second*time.Duration(shortDead+1) + nextScheduleDelta,
  1173  			expectUpdateStatus:         true,
  1174  			jobPresentInCJActiveStatus: true,
  1175  		},
  1176  		"do nothing if the namespace is terminating": {
  1177  			jobCreateError: &errors.StatusError{ErrStatus: metav1.Status{Details: &metav1.StatusDetails{Causes: []metav1.StatusCause{
  1178  				{
  1179  					Type:    v1.NamespaceTerminatingCause,
  1180  					Message: fmt.Sprintf("namespace %s is being terminated", metav1.NamespaceDefault),
  1181  					Field:   "metadata.namespace",
  1182  				}}}}},
  1183  			concurrencyPolicy:          "Allow",
  1184  			schedule:                   onTheHour,
  1185  			deadline:                   noDead,
  1186  			ranPreviously:              true,
  1187  			stillActive:                true,
  1188  			jobCreationTime:            justAfterThePriorHour(),
  1189  			now:                        *justAfterTheHour(),
  1190  			expectActive:               0,
  1191  			expectRequeueAfter:         false,
  1192  			expectUpdateStatus:         false,
  1193  			expectErr:                  true,
  1194  			jobPresentInCJActiveStatus: false,
  1195  		},
  1196  	}
  1197  	for name, tc := range testCases {
  1198  		name := name
  1199  		tc := tc
  1200  
  1201  		t.Run(name, func(t *testing.T) {
  1202  			cj := cronJob()
  1203  			cj.Spec.ConcurrencyPolicy = tc.concurrencyPolicy
  1204  			cj.Spec.Suspend = &tc.suspend
  1205  			cj.Spec.Schedule = tc.schedule
  1206  			cj.Spec.TimeZone = tc.timeZone
  1207  			if tc.deadline != noDead {
  1208  				cj.Spec.StartingDeadlineSeconds = &tc.deadline
  1209  			}
  1210  
  1211  			var (
  1212  				job *batchv1.Job
  1213  				err error
  1214  			)
  1215  			js := []*batchv1.Job{}
  1216  			realCJ := cj.DeepCopy()
  1217  			if tc.ranPreviously {
  1218  				cj.ObjectMeta.CreationTimestamp = metav1.Time{Time: justBeforeThePriorHour()}
  1219  				if !tc.cronjobCreationTime.IsZero() {
  1220  					cj.ObjectMeta.CreationTimestamp = metav1.Time{Time: tc.cronjobCreationTime}
  1221  				}
  1222  				cj.Status.LastScheduleTime = &metav1.Time{Time: justAfterThePriorHour()}
  1223  				if !tc.lastScheduleTime.IsZero() {
  1224  					cj.Status.LastScheduleTime = &metav1.Time{Time: tc.lastScheduleTime}
  1225  				}
  1226  				job, err = getJobFromTemplate2(&cj, tc.jobCreationTime)
  1227  				if err != nil {
  1228  					t.Fatalf("%s: unexpected error creating a job from template: %v", name, err)
  1229  				}
  1230  				job.UID = "1234"
  1231  				job.Namespace = cj.Namespace
  1232  				if tc.stillActive {
  1233  					ref, err := getRef(job)
  1234  					if err != nil {
  1235  						t.Fatalf("%s: unexpected error getting the job object reference: %v", name, err)
  1236  					}
  1237  					if tc.jobPresentInCJActiveStatus {
  1238  						cj.Status.Active = []v1.ObjectReference{*ref}
  1239  					}
  1240  					realCJ.Status.Active = []v1.ObjectReference{*ref}
  1241  					if !tc.jobStillNotFoundInLister {
  1242  						js = append(js, job)
  1243  					}
  1244  				} else {
  1245  					job.Status.CompletionTime = &metav1.Time{Time: job.ObjectMeta.CreationTimestamp.Add(time.Second * 10)}
  1246  					job.Status.Conditions = append(job.Status.Conditions, batchv1.JobCondition{
  1247  						Type:   batchv1.JobComplete,
  1248  						Status: v1.ConditionTrue,
  1249  					})
  1250  					if !tc.jobStillNotFoundInLister {
  1251  						js = append(js, job)
  1252  					}
  1253  				}
  1254  			} else {
  1255  				cj.ObjectMeta.CreationTimestamp = metav1.Time{Time: justBeforeTheHour()}
  1256  				if !tc.cronjobCreationTime.IsZero() {
  1257  					cj.ObjectMeta.CreationTimestamp = metav1.Time{Time: tc.cronjobCreationTime}
  1258  				}
  1259  				if tc.stillActive {
  1260  					t.Errorf("%s: test setup error: this case makes no sense", name)
  1261  				}
  1262  			}
  1263  
  1264  			jc := &fakeJobControl{Job: job, CreateErr: tc.jobCreateError, Err: tc.jobGetErr}
  1265  			cjc := &fakeCJControl{CronJob: realCJ}
  1266  			recorder := record.NewFakeRecorder(10)
  1267  
  1268  			jm := ControllerV2{
  1269  				jobControl:     jc,
  1270  				cronJobControl: cjc,
  1271  				recorder:       recorder,
  1272  				now: func() time.Time {
  1273  					return tc.now
  1274  				},
  1275  			}
  1276  			cjCopy := cj.DeepCopy()
  1277  			requeueAfter, updateStatus, err := jm.syncCronJob(context.TODO(), cjCopy, js)
  1278  			if tc.expectErr && err == nil {
  1279  				t.Errorf("%s: expected error got none with requeueAfter time: %#v", name, requeueAfter)
  1280  			}
  1281  			if tc.expectRequeueAfter {
  1282  				if !reflect.DeepEqual(requeueAfter, &tc.expectedRequeueDuration) {
  1283  					t.Errorf("%s: expected requeueAfter: %+v, got requeueAfter time: %+v", name, tc.expectedRequeueDuration, requeueAfter)
  1284  				}
  1285  			}
  1286  			if updateStatus != tc.expectUpdateStatus {
  1287  				t.Errorf("%s: expected updateStatus: %t, actually: %t", name, tc.expectUpdateStatus, updateStatus)
  1288  			}
  1289  			expectedCreates := 0
  1290  			if tc.expectCreate {
  1291  				expectedCreates = 1
  1292  			}
  1293  			if tc.ranPreviously && !tc.stillActive {
  1294  				completionTime := tc.jobCreationTime.Add(10 * time.Second)
  1295  				if cjCopy.Status.LastSuccessfulTime == nil || !cjCopy.Status.LastSuccessfulTime.Time.Equal(completionTime) {
  1296  					t.Errorf("cj.status.lastSuccessfulTime: %s expected, got %#v", completionTime, cj.Status.LastSuccessfulTime)
  1297  				}
  1298  			}
  1299  			if len(jc.Jobs) != expectedCreates {
  1300  				t.Errorf("%s: expected %d job started, actually %v", name, expectedCreates, len(jc.Jobs))
  1301  			}
  1302  			for i := range jc.Jobs {
  1303  				job := &jc.Jobs[i]
  1304  				controllerRef := metav1.GetControllerOf(job)
  1305  				if controllerRef == nil {
  1306  					t.Errorf("%s: expected job to have ControllerRef: %#v", name, job)
  1307  				} else {
  1308  					if got, want := controllerRef.APIVersion, "batch/v1"; got != want {
  1309  						t.Errorf("%s: controllerRef.APIVersion = %q, want %q", name, got, want)
  1310  					}
  1311  					if got, want := controllerRef.Kind, "CronJob"; got != want {
  1312  						t.Errorf("%s: controllerRef.Kind = %q, want %q", name, got, want)
  1313  					}
  1314  					if got, want := controllerRef.Name, cj.Name; got != want {
  1315  						t.Errorf("%s: controllerRef.Name = %q, want %q", name, got, want)
  1316  					}
  1317  					if got, want := controllerRef.UID, cj.UID; got != want {
  1318  						t.Errorf("%s: controllerRef.UID = %q, want %q", name, got, want)
  1319  					}
  1320  					if controllerRef.Controller == nil || *controllerRef.Controller != true {
  1321  						t.Errorf("%s: controllerRef.Controller is not set to true", name)
  1322  					}
  1323  				}
  1324  			}
  1325  
  1326  			expectedDeletes := 0
  1327  			if tc.expectDelete {
  1328  				expectedDeletes = 1
  1329  			}
  1330  			if len(jc.DeleteJobName) != expectedDeletes {
  1331  				t.Errorf("%s: expected %d job deleted, actually %v", name, expectedDeletes, len(jc.DeleteJobName))
  1332  			}
  1333  
  1334  			// Status update happens once when ranging through job list, and another one if create jobs.
  1335  			expectUpdates := 1
  1336  			expectedEvents := 0
  1337  			if tc.expectCreate {
  1338  				expectedEvents++
  1339  				expectUpdates++
  1340  			}
  1341  			if tc.expectDelete {
  1342  				expectedEvents++
  1343  			}
  1344  			if name == "still active, is time, F" {
  1345  				// this is the only test case where we would raise an event for not scheduling
  1346  				expectedEvents++
  1347  			}
  1348  			expectedEvents += tc.expectedWarnings
  1349  
  1350  			if len(recorder.Events) != expectedEvents {
  1351  				t.Errorf("%s: expected %d event, actually %v", name, expectedEvents, len(recorder.Events))
  1352  			}
  1353  
  1354  			numWarnings := 0
  1355  			for i := 1; i <= len(recorder.Events); i++ {
  1356  				e := <-recorder.Events
  1357  				if strings.HasPrefix(e, v1.EventTypeWarning) {
  1358  					numWarnings++
  1359  				}
  1360  			}
  1361  			if numWarnings != tc.expectedWarnings {
  1362  				t.Errorf("%s: expected %d warnings, actually %v", name, tc.expectedWarnings, numWarnings)
  1363  			}
  1364  
  1365  			if len(cjc.Updates) == expectUpdates && tc.expectActive != len(cjc.Updates[expectUpdates-1].Status.Active) {
  1366  				t.Errorf("%s: expected Active size %d, got %d", name, tc.expectActive, len(cjc.Updates[expectUpdates-1].Status.Active))
  1367  			}
  1368  
  1369  			if &cj == cjCopy {
  1370  				t.Errorf("syncCronJob is not creating a copy of the original cronjob")
  1371  			}
  1372  		})
  1373  	}
  1374  
  1375  }
  1376  
  1377  type fakeQueue struct {
  1378  	workqueue.RateLimitingInterface
  1379  	delay time.Duration
  1380  	key   interface{}
  1381  }
  1382  
  1383  func (f *fakeQueue) AddAfter(key interface{}, delay time.Duration) {
  1384  	f.delay = delay
  1385  	f.key = key
  1386  }
  1387  
  1388  // this test will take around 61 seconds to complete
  1389  func TestControllerV2UpdateCronJob(t *testing.T) {
  1390  	tests := []struct {
  1391  		name          string
  1392  		oldCronJob    *batchv1.CronJob
  1393  		newCronJob    *batchv1.CronJob
  1394  		expectedDelay time.Duration
  1395  	}{
  1396  		{
  1397  			name: "spec.template changed",
  1398  			oldCronJob: &batchv1.CronJob{
  1399  				Spec: batchv1.CronJobSpec{
  1400  					JobTemplate: batchv1.JobTemplateSpec{
  1401  						ObjectMeta: metav1.ObjectMeta{
  1402  							Labels:      map[string]string{"a": "b"},
  1403  							Annotations: map[string]string{"x": "y"},
  1404  						},
  1405  						Spec: jobSpec(),
  1406  					},
  1407  				},
  1408  			},
  1409  			newCronJob: &batchv1.CronJob{
  1410  				Spec: batchv1.CronJobSpec{
  1411  					JobTemplate: batchv1.JobTemplateSpec{
  1412  						ObjectMeta: metav1.ObjectMeta{
  1413  							Labels:      map[string]string{"a": "foo"},
  1414  							Annotations: map[string]string{"x": "y"},
  1415  						},
  1416  						Spec: jobSpec(),
  1417  					},
  1418  				},
  1419  			},
  1420  			expectedDelay: 0 * time.Second,
  1421  		},
  1422  		{
  1423  			name: "spec.schedule changed",
  1424  			oldCronJob: &batchv1.CronJob{
  1425  				Spec: batchv1.CronJobSpec{
  1426  					Schedule: "30 * * * *",
  1427  					JobTemplate: batchv1.JobTemplateSpec{
  1428  						ObjectMeta: metav1.ObjectMeta{
  1429  							Labels:      map[string]string{"a": "b"},
  1430  							Annotations: map[string]string{"x": "y"},
  1431  						},
  1432  						Spec: jobSpec(),
  1433  					},
  1434  				},
  1435  				Status: batchv1.CronJobStatus{
  1436  					LastScheduleTime: &metav1.Time{Time: justBeforeTheHour()},
  1437  				},
  1438  			},
  1439  			newCronJob: &batchv1.CronJob{
  1440  				Spec: batchv1.CronJobSpec{
  1441  					Schedule: "*/1 * * * *",
  1442  					JobTemplate: batchv1.JobTemplateSpec{
  1443  						ObjectMeta: metav1.ObjectMeta{
  1444  							Labels:      map[string]string{"a": "foo"},
  1445  							Annotations: map[string]string{"x": "y"},
  1446  						},
  1447  						Spec: jobSpec(),
  1448  					},
  1449  				},
  1450  				Status: batchv1.CronJobStatus{
  1451  					LastScheduleTime: &metav1.Time{Time: justBeforeTheHour()},
  1452  				},
  1453  			},
  1454  			expectedDelay: 1*time.Second + nextScheduleDelta,
  1455  		},
  1456  		{
  1457  			name: "spec.schedule with @every changed - cadence decrease",
  1458  			oldCronJob: &batchv1.CronJob{
  1459  				Spec: batchv1.CronJobSpec{
  1460  					Schedule: "@every 1m",
  1461  					JobTemplate: batchv1.JobTemplateSpec{
  1462  						ObjectMeta: metav1.ObjectMeta{
  1463  							Labels:      map[string]string{"a": "b"},
  1464  							Annotations: map[string]string{"x": "y"},
  1465  						},
  1466  						Spec: jobSpec(),
  1467  					},
  1468  				},
  1469  				Status: batchv1.CronJobStatus{
  1470  					LastScheduleTime: &metav1.Time{Time: justBeforeTheHour()},
  1471  				},
  1472  			},
  1473  			newCronJob: &batchv1.CronJob{
  1474  				Spec: batchv1.CronJobSpec{
  1475  					Schedule: "@every 3m",
  1476  					JobTemplate: batchv1.JobTemplateSpec{
  1477  						ObjectMeta: metav1.ObjectMeta{
  1478  							Labels:      map[string]string{"a": "foo"},
  1479  							Annotations: map[string]string{"x": "y"},
  1480  						},
  1481  						Spec: jobSpec(),
  1482  					},
  1483  				},
  1484  				Status: batchv1.CronJobStatus{
  1485  					LastScheduleTime: &metav1.Time{Time: justBeforeTheHour()},
  1486  				},
  1487  			},
  1488  			expectedDelay: 2*time.Minute + 1*time.Second + nextScheduleDelta,
  1489  		},
  1490  		{
  1491  			name: "spec.schedule with @every changed - cadence increase",
  1492  			oldCronJob: &batchv1.CronJob{
  1493  				Spec: batchv1.CronJobSpec{
  1494  					Schedule: "@every 3m",
  1495  					JobTemplate: batchv1.JobTemplateSpec{
  1496  						ObjectMeta: metav1.ObjectMeta{
  1497  							Labels:      map[string]string{"a": "b"},
  1498  							Annotations: map[string]string{"x": "y"},
  1499  						},
  1500  						Spec: jobSpec(),
  1501  					},
  1502  				},
  1503  				Status: batchv1.CronJobStatus{
  1504  					LastScheduleTime: &metav1.Time{Time: justBeforeTheHour()},
  1505  				},
  1506  			},
  1507  			newCronJob: &batchv1.CronJob{
  1508  				Spec: batchv1.CronJobSpec{
  1509  					Schedule: "@every 1m",
  1510  					JobTemplate: batchv1.JobTemplateSpec{
  1511  						ObjectMeta: metav1.ObjectMeta{
  1512  							Labels:      map[string]string{"a": "foo"},
  1513  							Annotations: map[string]string{"x": "y"},
  1514  						},
  1515  						Spec: jobSpec(),
  1516  					},
  1517  				},
  1518  				Status: batchv1.CronJobStatus{
  1519  					LastScheduleTime: &metav1.Time{Time: justBeforeTheHour()},
  1520  				},
  1521  			},
  1522  			expectedDelay: 1*time.Second + nextScheduleDelta,
  1523  		},
  1524  		{
  1525  			name: "spec.timeZone not changed",
  1526  			oldCronJob: &batchv1.CronJob{
  1527  				Spec: batchv1.CronJobSpec{
  1528  					TimeZone: &newYork,
  1529  					JobTemplate: batchv1.JobTemplateSpec{
  1530  						ObjectMeta: metav1.ObjectMeta{
  1531  							Labels:      map[string]string{"a": "b"},
  1532  							Annotations: map[string]string{"x": "y"},
  1533  						},
  1534  						Spec: jobSpec(),
  1535  					},
  1536  				},
  1537  			},
  1538  			newCronJob: &batchv1.CronJob{
  1539  				Spec: batchv1.CronJobSpec{
  1540  					TimeZone: &newYork,
  1541  					JobTemplate: batchv1.JobTemplateSpec{
  1542  						ObjectMeta: metav1.ObjectMeta{
  1543  							Labels:      map[string]string{"a": "foo"},
  1544  							Annotations: map[string]string{"x": "y"},
  1545  						},
  1546  						Spec: jobSpec(),
  1547  					},
  1548  				},
  1549  			},
  1550  			expectedDelay: 0 * time.Second,
  1551  		},
  1552  		{
  1553  			name: "spec.timeZone changed",
  1554  			oldCronJob: &batchv1.CronJob{
  1555  				Spec: batchv1.CronJobSpec{
  1556  					TimeZone: &newYork,
  1557  					JobTemplate: batchv1.JobTemplateSpec{
  1558  						ObjectMeta: metav1.ObjectMeta{
  1559  							Labels:      map[string]string{"a": "b"},
  1560  							Annotations: map[string]string{"x": "y"},
  1561  						},
  1562  						Spec: jobSpec(),
  1563  					},
  1564  				},
  1565  			},
  1566  			newCronJob: &batchv1.CronJob{
  1567  				Spec: batchv1.CronJobSpec{
  1568  					TimeZone: nil,
  1569  					JobTemplate: batchv1.JobTemplateSpec{
  1570  						ObjectMeta: metav1.ObjectMeta{
  1571  							Labels:      map[string]string{"a": "foo"},
  1572  							Annotations: map[string]string{"x": "y"},
  1573  						},
  1574  						Spec: jobSpec(),
  1575  					},
  1576  				},
  1577  			},
  1578  			expectedDelay: 0 * time.Second,
  1579  		},
  1580  
  1581  		// TODO: Add more test cases for updating scheduling.
  1582  	}
  1583  	for _, tt := range tests {
  1584  		t.Run(tt.name, func(t *testing.T) {
  1585  			logger, ctx := ktesting.NewTestContext(t)
  1586  			ctx, cancel := context.WithCancel(ctx)
  1587  			defer cancel()
  1588  			kubeClient := fake.NewSimpleClientset()
  1589  			sharedInformers := informers.NewSharedInformerFactory(kubeClient, controller.NoResyncPeriodFunc())
  1590  			jm, err := NewControllerV2(ctx, sharedInformers.Batch().V1().Jobs(), sharedInformers.Batch().V1().CronJobs(), kubeClient)
  1591  			if err != nil {
  1592  				t.Errorf("unexpected error %v", err)
  1593  				return
  1594  			}
  1595  			jm.now = justASecondBeforeTheHour
  1596  			queue := &fakeQueue{RateLimitingInterface: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "test-update-cronjob")}
  1597  			jm.queue = queue
  1598  			jm.jobControl = &fakeJobControl{}
  1599  			jm.cronJobControl = &fakeCJControl{}
  1600  			jm.recorder = record.NewFakeRecorder(10)
  1601  
  1602  			jm.updateCronJob(logger, tt.oldCronJob, tt.newCronJob)
  1603  			if queue.delay.Seconds() != tt.expectedDelay.Seconds() {
  1604  				t.Errorf("Expected delay %#v got %#v", tt.expectedDelay.Seconds(), queue.delay.Seconds())
  1605  			}
  1606  		})
  1607  	}
  1608  }
  1609  
  1610  func TestControllerV2GetJobsToBeReconciled(t *testing.T) {
  1611  	trueRef := true
  1612  	tests := []struct {
  1613  		name     string
  1614  		cronJob  *batchv1.CronJob
  1615  		jobs     []runtime.Object
  1616  		expected []*batchv1.Job
  1617  	}{
  1618  		{
  1619  			name:    "test getting jobs in namespace without controller reference",
  1620  			cronJob: &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Namespace: "foo-ns", Name: "fooer"}},
  1621  			jobs: []runtime.Object{
  1622  				&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "foo-ns"}},
  1623  				&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo1", Namespace: "foo-ns"}},
  1624  				&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo2", Namespace: "foo-ns"}},
  1625  			},
  1626  			expected: []*batchv1.Job{},
  1627  		},
  1628  		{
  1629  			name:    "test getting jobs in namespace with a controller reference",
  1630  			cronJob: &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Namespace: "foo-ns", Name: "fooer"}},
  1631  			jobs: []runtime.Object{
  1632  				&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "foo-ns"}},
  1633  				&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo1", Namespace: "foo-ns",
  1634  					OwnerReferences: []metav1.OwnerReference{{Name: "fooer", Controller: &trueRef}}}},
  1635  				&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo2", Namespace: "foo-ns"}},
  1636  			},
  1637  			expected: []*batchv1.Job{
  1638  				{ObjectMeta: metav1.ObjectMeta{Name: "foo1", Namespace: "foo-ns",
  1639  					OwnerReferences: []metav1.OwnerReference{{Name: "fooer", Controller: &trueRef}}}},
  1640  			},
  1641  		},
  1642  		{
  1643  			name:    "test getting jobs in other namespaces",
  1644  			cronJob: &batchv1.CronJob{ObjectMeta: metav1.ObjectMeta{Namespace: "foo-ns", Name: "fooer"}},
  1645  			jobs: []runtime.Object{
  1646  				&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "bar-ns"}},
  1647  				&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo1", Namespace: "bar-ns"}},
  1648  				&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "foo2", Namespace: "bar-ns"}},
  1649  			},
  1650  			expected: []*batchv1.Job{},
  1651  		},
  1652  		{
  1653  			name: "test getting jobs whose labels do not match job template",
  1654  			cronJob: &batchv1.CronJob{
  1655  				ObjectMeta: metav1.ObjectMeta{Namespace: "foo-ns", Name: "fooer"},
  1656  				Spec: batchv1.CronJobSpec{JobTemplate: batchv1.JobTemplateSpec{
  1657  					ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"key": "value"}},
  1658  				}},
  1659  			},
  1660  			jobs: []runtime.Object{
  1661  				&batchv1.Job{ObjectMeta: metav1.ObjectMeta{
  1662  					Namespace:       "foo-ns",
  1663  					Name:            "foo-fooer-owner-ref",
  1664  					Labels:          map[string]string{"key": "different-value"},
  1665  					OwnerReferences: []metav1.OwnerReference{{Name: "fooer", Controller: &trueRef}}},
  1666  				},
  1667  				&batchv1.Job{ObjectMeta: metav1.ObjectMeta{
  1668  					Namespace:       "foo-ns",
  1669  					Name:            "foo-other-owner-ref",
  1670  					Labels:          map[string]string{"key": "different-value"},
  1671  					OwnerReferences: []metav1.OwnerReference{{Name: "another-cronjob", Controller: &trueRef}}},
  1672  				},
  1673  			},
  1674  			expected: []*batchv1.Job{{
  1675  				ObjectMeta: metav1.ObjectMeta{
  1676  					Namespace:       "foo-ns",
  1677  					Name:            "foo-fooer-owner-ref",
  1678  					Labels:          map[string]string{"key": "different-value"},
  1679  					OwnerReferences: []metav1.OwnerReference{{Name: "fooer", Controller: &trueRef}}},
  1680  			}},
  1681  		},
  1682  	}
  1683  	for _, tt := range tests {
  1684  		t.Run(tt.name, func(t *testing.T) {
  1685  			_, ctx := ktesting.NewTestContext(t)
  1686  			ctx, cancel := context.WithCancel(ctx)
  1687  			defer cancel()
  1688  			kubeClient := fake.NewSimpleClientset()
  1689  			sharedInformers := informers.NewSharedInformerFactory(kubeClient, controller.NoResyncPeriodFunc())
  1690  			for _, job := range tt.jobs {
  1691  				sharedInformers.Batch().V1().Jobs().Informer().GetIndexer().Add(job)
  1692  			}
  1693  			jm, err := NewControllerV2(ctx, sharedInformers.Batch().V1().Jobs(), sharedInformers.Batch().V1().CronJobs(), kubeClient)
  1694  			if err != nil {
  1695  				t.Errorf("unexpected error %v", err)
  1696  				return
  1697  			}
  1698  
  1699  			actual, err := jm.getJobsToBeReconciled(tt.cronJob)
  1700  			if err != nil {
  1701  				t.Errorf("unexpected error %v", err)
  1702  				return
  1703  			}
  1704  			if !reflect.DeepEqual(actual, tt.expected) {
  1705  				t.Errorf("\nExpected %#v,\nbut got %#v", tt.expected, actual)
  1706  			}
  1707  		})
  1708  	}
  1709  }
  1710  
  1711  func TestControllerV2CleanupFinishedJobs(t *testing.T) {
  1712  	tests := []struct {
  1713  		name                string
  1714  		now                 time.Time
  1715  		cronJob             *batchv1.CronJob
  1716  		finishedJobs        []*batchv1.Job
  1717  		jobCreateError      error
  1718  		expectedDeletedJobs []string
  1719  	}{
  1720  		{
  1721  			name: "jobs are still deleted when a cronjob can't create jobs due to jobs quota being reached (avoiding a deadlock)",
  1722  			now:  *justAfterTheHour(),
  1723  			cronJob: &batchv1.CronJob{
  1724  				ObjectMeta: metav1.ObjectMeta{Namespace: "foo-ns", Name: "fooer"},
  1725  				Spec: batchv1.CronJobSpec{
  1726  					Schedule:                   onTheHour,
  1727  					SuccessfulJobsHistoryLimit: pointer.Int32(1),
  1728  					JobTemplate: batchv1.JobTemplateSpec{
  1729  						ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"key": "value"}},
  1730  					},
  1731  				},
  1732  				Status: batchv1.CronJobStatus{LastScheduleTime: &metav1.Time{Time: justAfterThePriorHour()}},
  1733  			},
  1734  			finishedJobs: []*batchv1.Job{
  1735  				{
  1736  					ObjectMeta: metav1.ObjectMeta{
  1737  						Namespace:       "foo-ns",
  1738  						Name:            "finished-job-started-hour-ago",
  1739  						OwnerReferences: []metav1.OwnerReference{{Name: "fooer", Controller: pointer.Bool(true)}},
  1740  					},
  1741  					Status: batchv1.JobStatus{StartTime: &metav1.Time{Time: justBeforeThePriorHour()}},
  1742  				},
  1743  				{
  1744  					ObjectMeta: metav1.ObjectMeta{
  1745  						Namespace:       "foo-ns",
  1746  						Name:            "finished-job-started-minute-ago",
  1747  						OwnerReferences: []metav1.OwnerReference{{Name: "fooer", Controller: pointer.Bool(true)}},
  1748  					},
  1749  					Status: batchv1.JobStatus{StartTime: &metav1.Time{Time: justBeforeTheHour()}},
  1750  				},
  1751  			},
  1752  			jobCreateError:      errors.NewInternalError(fmt.Errorf("quota for # of jobs reached")),
  1753  			expectedDeletedJobs: []string{"finished-job-started-hour-ago"},
  1754  		},
  1755  		{
  1756  			name: "jobs are not deleted if history limit not reached",
  1757  			now:  justBeforeTheHour(),
  1758  			cronJob: &batchv1.CronJob{
  1759  				ObjectMeta: metav1.ObjectMeta{Namespace: "foo-ns", Name: "fooer"},
  1760  				Spec: batchv1.CronJobSpec{
  1761  					Schedule:                   onTheHour,
  1762  					SuccessfulJobsHistoryLimit: pointer.Int32(2),
  1763  					JobTemplate: batchv1.JobTemplateSpec{
  1764  						ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"key": "value"}},
  1765  					},
  1766  				},
  1767  				Status: batchv1.CronJobStatus{LastScheduleTime: &metav1.Time{Time: justAfterThePriorHour()}},
  1768  			},
  1769  			finishedJobs: []*batchv1.Job{
  1770  				{
  1771  					ObjectMeta: metav1.ObjectMeta{
  1772  						Namespace:       "foo-ns",
  1773  						Name:            "finished-job-started-hour-ago",
  1774  						OwnerReferences: []metav1.OwnerReference{{Name: "fooer", Controller: pointer.Bool(true)}},
  1775  					},
  1776  					Status: batchv1.JobStatus{StartTime: &metav1.Time{Time: justBeforeThePriorHour()}},
  1777  				},
  1778  			},
  1779  			jobCreateError:      nil,
  1780  			expectedDeletedJobs: []string{},
  1781  		},
  1782  	}
  1783  
  1784  	for _, tt := range tests {
  1785  		t.Run(tt.name, func(t *testing.T) {
  1786  			_, ctx := ktesting.NewTestContext(t)
  1787  
  1788  			for _, job := range tt.finishedJobs {
  1789  				job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: v1.ConditionTrue}}
  1790  			}
  1791  
  1792  			client := fake.NewSimpleClientset()
  1793  
  1794  			informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc())
  1795  			_ = informerFactory.Batch().V1().CronJobs().Informer().GetIndexer().Add(tt.cronJob)
  1796  			for _, job := range tt.finishedJobs {
  1797  				_ = informerFactory.Batch().V1().Jobs().Informer().GetIndexer().Add(job)
  1798  			}
  1799  
  1800  			jm, err := NewControllerV2(ctx, informerFactory.Batch().V1().Jobs(), informerFactory.Batch().V1().CronJobs(), client)
  1801  			if err != nil {
  1802  				t.Errorf("unexpected error %v", err)
  1803  				return
  1804  			}
  1805  			jobControl := &fakeJobControl{CreateErr: tt.jobCreateError}
  1806  			jm.jobControl = jobControl
  1807  			jm.now = func() time.Time {
  1808  				return tt.now
  1809  			}
  1810  
  1811  			jm.enqueueController(tt.cronJob)
  1812  			jm.processNextWorkItem(ctx)
  1813  
  1814  			if len(tt.expectedDeletedJobs) != len(jobControl.DeleteJobName) {
  1815  				t.Fatalf("expected '%v' jobs to be deleted, instead deleted '%s'", tt.expectedDeletedJobs, jobControl.DeleteJobName)
  1816  			}
  1817  			sort.Strings(jobControl.DeleteJobName)
  1818  			sort.Strings(tt.expectedDeletedJobs)
  1819  			for i, deletedJob := range jobControl.DeleteJobName {
  1820  				if deletedJob != tt.expectedDeletedJobs[i] {
  1821  					t.Fatalf("expected '%v' jobs to be deleted, instead deleted '%s'", tt.expectedDeletedJobs, jobControl.DeleteJobName)
  1822  				}
  1823  			}
  1824  		})
  1825  	}
  1826  }
  1827  
  1828  // TestControllerV2JobAlreadyExistsButNotInActiveStatus validates that an already created job that was not added to the status
  1829  // of a CronJob initially will be added back on the next sync. Previously, if we failed to update the status after creating a job,
  1830  // cronjob controller would retry continuously because it would attempt to create a job that already exists.
  1831  func TestControllerV2JobAlreadyExistsButNotInActiveStatus(t *testing.T) {
  1832  	_, ctx := ktesting.NewTestContext(t)
  1833  
  1834  	cj := cronJob()
  1835  	cj.Spec.ConcurrencyPolicy = "Forbid"
  1836  	cj.Spec.Schedule = everyHour
  1837  	cj.Status.LastScheduleTime = &metav1.Time{Time: justBeforeThePriorHour()}
  1838  	cj.Status.Active = []v1.ObjectReference{}
  1839  	cjCopy := cj.DeepCopy()
  1840  
  1841  	job, err := getJobFromTemplate2(&cj, justAfterThePriorHour())
  1842  	if err != nil {
  1843  		t.Fatalf("Unexpected error creating a job from template: %v", err)
  1844  	}
  1845  	job.UID = "1234"
  1846  	job.Namespace = cj.Namespace
  1847  
  1848  	client := fake.NewSimpleClientset(cjCopy, job)
  1849  	informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc())
  1850  	_ = informerFactory.Batch().V1().CronJobs().Informer().GetIndexer().Add(cjCopy)
  1851  
  1852  	jm, err := NewControllerV2(ctx, informerFactory.Batch().V1().Jobs(), informerFactory.Batch().V1().CronJobs(), client)
  1853  	if err != nil {
  1854  		t.Fatalf("unexpected error %v", err)
  1855  	}
  1856  
  1857  	jobControl := &fakeJobControl{Job: job, CreateErr: errors.NewAlreadyExists(schema.GroupResource{Resource: "job", Group: "batch"}, "")}
  1858  	jm.jobControl = jobControl
  1859  	cronJobControl := &fakeCJControl{}
  1860  	jm.cronJobControl = cronJobControl
  1861  	jm.now = justBeforeTheHour
  1862  
  1863  	jm.enqueueController(cjCopy)
  1864  	jm.processNextWorkItem(ctx)
  1865  
  1866  	if len(cronJobControl.Updates) != 1 {
  1867  		t.Fatalf("Unexpected updates to cronjob, got: %d, expected 1", len(cronJobControl.Updates))
  1868  	}
  1869  	if len(cronJobControl.Updates[0].Status.Active) != 1 {
  1870  		t.Errorf("Unexpected active jobs count, got: %d, expected 1", len(cronJobControl.Updates[0].Status.Active))
  1871  	}
  1872  
  1873  	expectedActiveRef, err := getRef(job)
  1874  	if err != nil {
  1875  		t.Fatalf("Error getting expected job ref: %v", err)
  1876  	}
  1877  	if !reflect.DeepEqual(cronJobControl.Updates[0].Status.Active[0], *expectedActiveRef) {
  1878  		t.Errorf("Unexpected job reference in cronjob active list, got: %v, expected: %v", cronJobControl.Updates[0].Status.Active[0], expectedActiveRef)
  1879  	}
  1880  }
  1881  
  1882  // TestControllerV2JobAlreadyExistsButDifferentOwnner validates that an already created job
  1883  // not owned by the cronjob controller is ignored.
  1884  func TestControllerV2JobAlreadyExistsButDifferentOwner(t *testing.T) {
  1885  	_, ctx := ktesting.NewTestContext(t)
  1886  
  1887  	cj := cronJob()
  1888  	cj.Spec.ConcurrencyPolicy = "Forbid"
  1889  	cj.Spec.Schedule = everyHour
  1890  	cj.Status.LastScheduleTime = &metav1.Time{Time: justBeforeThePriorHour()}
  1891  	cj.Status.Active = []v1.ObjectReference{}
  1892  	cjCopy := cj.DeepCopy()
  1893  
  1894  	job, err := getJobFromTemplate2(&cj, justAfterThePriorHour())
  1895  	if err != nil {
  1896  		t.Fatalf("Unexpected error creating a job from template: %v", err)
  1897  	}
  1898  	job.UID = "1234"
  1899  	job.Namespace = cj.Namespace
  1900  
  1901  	// remove owners for this test since we are testing that jobs not belonging to cronjob
  1902  	// controller are safely ignored
  1903  	job.OwnerReferences = []metav1.OwnerReference{}
  1904  
  1905  	client := fake.NewSimpleClientset(cjCopy, job)
  1906  	informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc())
  1907  	_ = informerFactory.Batch().V1().CronJobs().Informer().GetIndexer().Add(cjCopy)
  1908  
  1909  	jm, err := NewControllerV2(ctx, informerFactory.Batch().V1().Jobs(), informerFactory.Batch().V1().CronJobs(), client)
  1910  	if err != nil {
  1911  		t.Fatalf("unexpected error %v", err)
  1912  	}
  1913  
  1914  	jobControl := &fakeJobControl{Job: job, CreateErr: errors.NewAlreadyExists(schema.GroupResource{Resource: "job", Group: "batch"}, "")}
  1915  	jm.jobControl = jobControl
  1916  	cronJobControl := &fakeCJControl{}
  1917  	jm.cronJobControl = cronJobControl
  1918  	jm.now = justBeforeTheHour
  1919  
  1920  	jm.enqueueController(cjCopy)
  1921  	jm.processNextWorkItem(ctx)
  1922  
  1923  	if len(cronJobControl.Updates) != 0 {
  1924  		t.Fatalf("Unexpected updates to cronjob, got: %d, expected 0", len(cronJobControl.Updates))
  1925  	}
  1926  }
  1927  

View as plain text