/* Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package job import ( "testing" "time" "github.com/google/go-cmp/cmp" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2/ktesting" clocktesting "k8s.io/utils/clock/testing" "k8s.io/utils/ptr" ) func TestNewBackoffRecord(t *testing.T) { emptyStoreInitializer := func(*backoffStore) {} defaultTestTime := metav1.NewTime(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)) testCases := map[string]struct { storeInitializer func(*backoffStore) uncounted uncountedTerminatedPods newSucceededPods []metav1.Time newFailedPods []metav1.Time wantBackoffRecord backoffRecord }{ "Empty backoff store and one new failure": { storeInitializer: emptyStoreInitializer, newSucceededPods: []metav1.Time{}, newFailedPods: []metav1.Time{ defaultTestTime, }, wantBackoffRecord: backoffRecord{ key: "key", lastFailureTime: &defaultTestTime.Time, failuresAfterLastSuccess: 1, }, }, "Empty backoff store and two new failures": { storeInitializer: emptyStoreInitializer, newSucceededPods: []metav1.Time{}, newFailedPods: []metav1.Time{ defaultTestTime, metav1.NewTime(defaultTestTime.Add(-1 * time.Millisecond)), }, wantBackoffRecord: backoffRecord{ key: "key", lastFailureTime: &defaultTestTime.Time, failuresAfterLastSuccess: 2, }, }, "Empty backoff store, two failures followed by success": { storeInitializer: emptyStoreInitializer, newSucceededPods: []metav1.Time{ defaultTestTime, }, newFailedPods: []metav1.Time{ metav1.NewTime(defaultTestTime.Add(-2 * time.Millisecond)), metav1.NewTime(defaultTestTime.Add(-1 * time.Millisecond)), }, wantBackoffRecord: backoffRecord{ key: "key", failuresAfterLastSuccess: 0, }, }, "Empty backoff store, two failures, one success and two more failures": { storeInitializer: emptyStoreInitializer, newSucceededPods: []metav1.Time{ metav1.NewTime(defaultTestTime.Add(-2 * time.Millisecond)), }, newFailedPods: []metav1.Time{ defaultTestTime, metav1.NewTime(defaultTestTime.Add(-4 * time.Millisecond)), metav1.NewTime(defaultTestTime.Add(-3 * time.Millisecond)), metav1.NewTime(defaultTestTime.Add(-1 * time.Millisecond)), }, wantBackoffRecord: backoffRecord{ key: "key", lastFailureTime: &defaultTestTime.Time, failuresAfterLastSuccess: 2, }, }, "Backoff store having failure count 2 and one new failure": { storeInitializer: func(bis *backoffStore) { bis.updateBackoffRecord(backoffRecord{ key: "key", failuresAfterLastSuccess: 2, lastFailureTime: nil, }) }, newSucceededPods: []metav1.Time{}, newFailedPods: []metav1.Time{ defaultTestTime, }, wantBackoffRecord: backoffRecord{ key: "key", lastFailureTime: &defaultTestTime.Time, failuresAfterLastSuccess: 3, }, }, "Empty backoff store with success and failure at same timestamp": { storeInitializer: emptyStoreInitializer, newSucceededPods: []metav1.Time{ defaultTestTime, }, newFailedPods: []metav1.Time{ defaultTestTime, }, wantBackoffRecord: backoffRecord{ key: "key", failuresAfterLastSuccess: 0, }, }, "Empty backoff store with no success/failure": { storeInitializer: emptyStoreInitializer, newSucceededPods: []metav1.Time{}, newFailedPods: []metav1.Time{}, wantBackoffRecord: backoffRecord{ key: "key", failuresAfterLastSuccess: 0, }, }, "Empty backoff store with one success": { storeInitializer: emptyStoreInitializer, newSucceededPods: []metav1.Time{ defaultTestTime, }, newFailedPods: []metav1.Time{}, wantBackoffRecord: backoffRecord{ key: "key", failuresAfterLastSuccess: 0, }, }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { backoffRecordStore := newBackoffStore() tc.storeInitializer(backoffRecordStore) newSucceededPods := []*v1.Pod{} newFailedPods := []*v1.Pod{} for _, finishTime := range tc.newSucceededPods { newSucceededPods = append(newSucceededPods, &v1.Pod{ ObjectMeta: metav1.ObjectMeta{}, Status: v1.PodStatus{ Phase: v1.PodSucceeded, ContainerStatuses: []v1.ContainerStatus{ { State: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ FinishedAt: finishTime, }, }, }, }, }, }) } for _, finishTime := range tc.newFailedPods { newFailedPods = append(newFailedPods, &v1.Pod{ ObjectMeta: metav1.ObjectMeta{}, Status: v1.PodStatus{ Phase: v1.PodFailed, ContainerStatuses: []v1.ContainerStatus{ { State: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{ FinishedAt: finishTime, }, }, }, }, }, }) } backoffRecord := backoffRecordStore.newBackoffRecord("key", newSucceededPods, newFailedPods) if diff := cmp.Diff(tc.wantBackoffRecord, backoffRecord, cmp.AllowUnexported(backoffRecord)); diff != "" { t.Errorf("backoffRecord not matching; (-want,+got): %v", diff) } }) } } func TestGetFinishedTime(t *testing.T) { defaultTestTime := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC) defaultTestTimeMinus30s := defaultTestTime.Add(-30 * time.Second) testCases := map[string]struct { pod v1.Pod wantFinishTime time.Time }{ "Pod with multiple containers and all containers terminated": { pod: v1.Pod{ Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { State: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{FinishedAt: metav1.NewTime(defaultTestTime.Add(-1 * time.Second))}, }, }, { State: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{FinishedAt: metav1.NewTime(defaultTestTime)}, }, }, { State: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{FinishedAt: metav1.NewTime(defaultTestTime.Add(-2 * time.Second))}, }, }, }, }, }, wantFinishTime: defaultTestTime, }, "Pod with multiple containers; two containers in terminated state and one in running state; fallback to deletionTimestamp": { pod: v1.Pod{ Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { State: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{FinishedAt: metav1.NewTime(defaultTestTime.Add(-1 * time.Second))}, }, }, { State: v1.ContainerState{ Running: &v1.ContainerStateRunning{}, }, }, { State: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{FinishedAt: metav1.NewTime(defaultTestTime.Add(-2 * time.Second))}, }, }, }, }, ObjectMeta: metav1.ObjectMeta{ DeletionTimestamp: &metav1.Time{Time: defaultTestTime}, }, }, wantFinishTime: defaultTestTime, }, "fallback to deletionTimestamp": { pod: v1.Pod{ Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { State: v1.ContainerState{ Running: &v1.ContainerStateRunning{}, }, }, }, }, ObjectMeta: metav1.ObjectMeta{ DeletionTimestamp: &metav1.Time{Time: defaultTestTime}, }, }, wantFinishTime: defaultTestTime, }, "fallback to deletionTimestamp, decremented by grace period": { pod: v1.Pod{ Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { State: v1.ContainerState{ Running: &v1.ContainerStateRunning{}, }, }, }, }, ObjectMeta: metav1.ObjectMeta{ DeletionTimestamp: &metav1.Time{Time: defaultTestTime}, DeletionGracePeriodSeconds: ptr.To[int64](30), }, }, wantFinishTime: defaultTestTimeMinus30s, }, "fallback to PodReady.LastTransitionTime when status of the condition is False": { pod: v1.Pod{ Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { State: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{}, }, }, }, Conditions: []v1.PodCondition{ { Type: v1.PodReady, Status: v1.ConditionFalse, Reason: "PodFailed", LastTransitionTime: metav1.Time{Time: defaultTestTime}, }, }, }, }, wantFinishTime: defaultTestTime, }, "skip fallback to PodReady.LastTransitionTime when status of the condition is True": { pod: v1.Pod{ Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { State: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{}, }, }, }, Conditions: []v1.PodCondition{ { Type: v1.PodReady, Status: v1.ConditionTrue, LastTransitionTime: metav1.Time{Time: defaultTestTimeMinus30s}, }, }, }, ObjectMeta: metav1.ObjectMeta{ DeletionTimestamp: &metav1.Time{Time: defaultTestTime}, }, }, wantFinishTime: defaultTestTime, }, "fallback to creationTimestamp": { pod: v1.Pod{ Status: v1.PodStatus{ ContainerStatuses: []v1.ContainerStatus{ { State: v1.ContainerState{ Terminated: &v1.ContainerStateTerminated{}, }, }, }, }, ObjectMeta: metav1.ObjectMeta{ CreationTimestamp: metav1.Time{Time: defaultTestTime}, }, }, wantFinishTime: defaultTestTime, }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { f := getFinishedTime(&tc.pod) if !f.Equal(tc.wantFinishTime) { t.Errorf("Expected value of finishedTime %v; got %v", tc.wantFinishTime, f) } }) } } func TestGetRemainingBackoffTime(t *testing.T) { defaultTestTime := metav1.NewTime(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)) testCases := map[string]struct { backoffRecord backoffRecord currentTime time.Time maxBackoff time.Duration defaultBackoff time.Duration wantDuration time.Duration }{ "no failures": { backoffRecord: backoffRecord{ lastFailureTime: nil, failuresAfterLastSuccess: 0, }, defaultBackoff: 5 * time.Second, maxBackoff: 700 * time.Second, wantDuration: 0 * time.Second, }, "one failure; current time and failure time are same": { backoffRecord: backoffRecord{ lastFailureTime: &defaultTestTime.Time, failuresAfterLastSuccess: 1, }, currentTime: defaultTestTime.Time, defaultBackoff: 5 * time.Second, maxBackoff: 700 * time.Second, wantDuration: 5 * time.Second, }, "one failure; current time == 1 second + failure time": { backoffRecord: backoffRecord{ lastFailureTime: &defaultTestTime.Time, failuresAfterLastSuccess: 1, }, currentTime: defaultTestTime.Time.Add(time.Second), defaultBackoff: 5 * time.Second, maxBackoff: 700 * time.Second, wantDuration: 4 * time.Second, }, "one failure; current time == expected backoff time": { backoffRecord: backoffRecord{ lastFailureTime: &defaultTestTime.Time, failuresAfterLastSuccess: 1, }, currentTime: defaultTestTime.Time.Add(5 * time.Second), defaultBackoff: 5 * time.Second, maxBackoff: 700 * time.Second, wantDuration: 0 * time.Second, }, "one failure; current time == expected backoff time + 1 Second": { backoffRecord: backoffRecord{ lastFailureTime: &defaultTestTime.Time, failuresAfterLastSuccess: 1, }, currentTime: defaultTestTime.Time.Add(6 * time.Second), defaultBackoff: 5 * time.Second, maxBackoff: 700 * time.Second, wantDuration: 0 * time.Second, }, "three failures; current time and failure time are same": { backoffRecord: backoffRecord{ lastFailureTime: &defaultTestTime.Time, failuresAfterLastSuccess: 3, }, currentTime: defaultTestTime.Time, defaultBackoff: 5 * time.Second, maxBackoff: 700 * time.Second, wantDuration: 20 * time.Second, }, "eight failures; current time and failure time are same; backoff not exceeding maxBackoff": { backoffRecord: backoffRecord{ lastFailureTime: &defaultTestTime.Time, failuresAfterLastSuccess: 8, }, currentTime: defaultTestTime.Time, defaultBackoff: 5 * time.Second, maxBackoff: 700 * time.Second, wantDuration: 640 * time.Second, }, "nine failures; current time and failure time are same; backoff exceeding maxBackoff": { backoffRecord: backoffRecord{ lastFailureTime: &defaultTestTime.Time, failuresAfterLastSuccess: 9, }, currentTime: defaultTestTime.Time, defaultBackoff: 5 * time.Second, maxBackoff: 700 * time.Second, wantDuration: 700 * time.Second, }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { fakeClock := clocktesting.NewFakeClock(tc.currentTime.Truncate(time.Second)) d := tc.backoffRecord.getRemainingTime(fakeClock, tc.defaultBackoff, tc.maxBackoff) if d.Seconds() != tc.wantDuration.Seconds() { t.Errorf("Expected value of duration %v; got %v", tc.wantDuration, d) } }) } } func TestGetRemainingBackoffTimePerIndex(t *testing.T) { defaultTestTime := metav1.NewTime(time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)) testCases := map[string]struct { currentTime time.Time maxBackoff time.Duration defaultBackoff time.Duration lastFailedPod *v1.Pod wantDuration time.Duration }{ "no failures": { lastFailedPod: nil, defaultBackoff: 5 * time.Second, maxBackoff: 700 * time.Second, wantDuration: 0 * time.Second, }, "two prev failures; current time and failure time are same": { lastFailedPod: buildPod().phase(v1.PodFailed).indexFailureCount("2").customDeletionTimestamp(defaultTestTime.Time).Pod, currentTime: defaultTestTime.Time, defaultBackoff: 5 * time.Second, maxBackoff: 700 * time.Second, wantDuration: 20 * time.Second, }, "one prev failure counted and one ignored; current time and failure time are same": { lastFailedPod: buildPod().phase(v1.PodFailed).indexFailureCount("1").indexIgnoredFailureCount("1").customDeletionTimestamp(defaultTestTime.Time).Pod, currentTime: defaultTestTime.Time, defaultBackoff: 5 * time.Second, maxBackoff: 700 * time.Second, wantDuration: 20 * time.Second, }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { logger, _ := ktesting.NewTestContext(t) fakeClock := clocktesting.NewFakeClock(tc.currentTime.Truncate(time.Second)) d := getRemainingTimePerIndex(logger, fakeClock, tc.defaultBackoff, tc.maxBackoff, tc.lastFailedPod) if d.Seconds() != tc.wantDuration.Seconds() { t.Errorf("Expected value of duration %v; got %v", tc.wantDuration, d) } }) } }