progress.go

Documentation: k8s.io/kubernetes/pkg/controller/deployment

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package deployment
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"reflect"
    23  	"time"
    24  
    25  	apps "k8s.io/api/apps/v1"
    26  	"k8s.io/api/core/v1"
    27  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    28  	"k8s.io/klog/v2"
    29  	"k8s.io/kubernetes/pkg/controller/deployment/util"
    30  )
    31  
    32  // syncRolloutStatus updates the status of a deployment during a rollout. There are
    33  // cases this helper will run that cannot be prevented from the scaling detection,
    34  // for example a resync of the deployment after it was scaled up. In those cases,
    35  // we shouldn't try to estimate any progress.
    36  func (dc *DeploymentController) syncRolloutStatus(ctx context.Context, allRSs []*apps.ReplicaSet, newRS *apps.ReplicaSet, d *apps.Deployment) error {
    37  	newStatus := calculateStatus(allRSs, newRS, d)
    38  
    39  	// If there is no progressDeadlineSeconds set, remove any Progressing condition.
    40  	if !util.HasProgressDeadline(d) {
    41  		util.RemoveDeploymentCondition(&newStatus, apps.DeploymentProgressing)
    42  	}
    43  
    44  	// If there is only one replica set that is active then that means we are not running
    45  	// a new rollout and this is a resync where we don't need to estimate any progress.
    46  	// In such a case, we should simply not estimate any progress for this deployment.
    47  	currentCond := util.GetDeploymentCondition(d.Status, apps.DeploymentProgressing)
    48  	isCompleteDeployment := newStatus.Replicas == newStatus.UpdatedReplicas && currentCond != nil && currentCond.Reason == util.NewRSAvailableReason
    49  	// Check for progress only if there is a progress deadline set and the latest rollout
    50  	// hasn't completed yet.
    51  	if util.HasProgressDeadline(d) && !isCompleteDeployment {
    52  		switch {
    53  		case util.DeploymentComplete(d, &newStatus):
    54  			// Update the deployment conditions with a message for the new replica set that
    55  			// was successfully deployed. If the condition already exists, we ignore this update.
    56  			msg := fmt.Sprintf("Deployment %q has successfully progressed.", d.Name)
    57  			if newRS != nil {
    58  				msg = fmt.Sprintf("ReplicaSet %q has successfully progressed.", newRS.Name)
    59  			}
    60  			condition := util.NewDeploymentCondition(apps.DeploymentProgressing, v1.ConditionTrue, util.NewRSAvailableReason, msg)
    61  			util.SetDeploymentCondition(&newStatus, *condition)
    62  
    63  		case util.DeploymentProgressing(d, &newStatus):
    64  			// If there is any progress made, continue by not checking if the deployment failed. This
    65  			// behavior emulates the rolling updater progressDeadline check.
    66  			msg := fmt.Sprintf("Deployment %q is progressing.", d.Name)
    67  			if newRS != nil {
    68  				msg = fmt.Sprintf("ReplicaSet %q is progressing.", newRS.Name)
    69  			}
    70  			condition := util.NewDeploymentCondition(apps.DeploymentProgressing, v1.ConditionTrue, util.ReplicaSetUpdatedReason, msg)
    71  			// Update the current Progressing condition or add a new one if it doesn't exist.
    72  			// If a Progressing condition with status=true already exists, we should update
    73  			// everything but lastTransitionTime. SetDeploymentCondition already does that but
    74  			// it also is not updating conditions when the reason of the new condition is the
    75  			// same as the old. The Progressing condition is a special case because we want to
    76  			// update with the same reason and change just lastUpdateTime iff we notice any
    77  			// progress. That's why we handle it here.
    78  			if currentCond != nil {
    79  				if currentCond.Status == v1.ConditionTrue {
    80  					condition.LastTransitionTime = currentCond.LastTransitionTime
    81  				}
    82  				util.RemoveDeploymentCondition(&newStatus, apps.DeploymentProgressing)
    83  			}
    84  			util.SetDeploymentCondition(&newStatus, *condition)
    85  
    86  		case util.DeploymentTimedOut(ctx, d, &newStatus):
    87  			// Update the deployment with a timeout condition. If the condition already exists,
    88  			// we ignore this update.
    89  			msg := fmt.Sprintf("Deployment %q has timed out progressing.", d.Name)
    90  			if newRS != nil {
    91  				msg = fmt.Sprintf("ReplicaSet %q has timed out progressing.", newRS.Name)
    92  			}
    93  			condition := util.NewDeploymentCondition(apps.DeploymentProgressing, v1.ConditionFalse, util.TimedOutReason, msg)
    94  			util.SetDeploymentCondition(&newStatus, *condition)
    95  		}
    96  	}
    97  
    98  	// Move failure conditions of all replica sets in deployment conditions. For now,
    99  	// only one failure condition is returned from getReplicaFailures.
   100  	if replicaFailureCond := dc.getReplicaFailures(allRSs, newRS); len(replicaFailureCond) > 0 {
   101  		// There will be only one ReplicaFailure condition on the replica set.
   102  		util.SetDeploymentCondition(&newStatus, replicaFailureCond[0])
   103  	} else {
   104  		util.RemoveDeploymentCondition(&newStatus, apps.DeploymentReplicaFailure)
   105  	}
   106  
   107  	// Do not update if there is nothing new to add.
   108  	if reflect.DeepEqual(d.Status, newStatus) {
   109  		// Requeue the deployment if required.
   110  		dc.requeueStuckDeployment(ctx, d, newStatus)
   111  		return nil
   112  	}
   113  
   114  	newDeployment := d
   115  	newDeployment.Status = newStatus
   116  	_, err := dc.client.AppsV1().Deployments(newDeployment.Namespace).UpdateStatus(ctx, newDeployment, metav1.UpdateOptions{})
   117  	return err
   118  }
   119  
   120  // getReplicaFailures will convert replica failure conditions from replica sets
   121  // to deployment conditions.
   122  func (dc *DeploymentController) getReplicaFailures(allRSs []*apps.ReplicaSet, newRS *apps.ReplicaSet) []apps.DeploymentCondition {
   123  	var conditions []apps.DeploymentCondition
   124  	if newRS != nil {
   125  		for _, c := range newRS.Status.Conditions {
   126  			if c.Type != apps.ReplicaSetReplicaFailure {
   127  				continue
   128  			}
   129  			conditions = append(conditions, util.ReplicaSetToDeploymentCondition(c))
   130  		}
   131  	}
   132  
   133  	// Return failures for the new replica set over failures from old replica sets.
   134  	if len(conditions) > 0 {
   135  		return conditions
   136  	}
   137  
   138  	for i := range allRSs {
   139  		rs := allRSs[i]
   140  		if rs == nil {
   141  			continue
   142  		}
   143  
   144  		for _, c := range rs.Status.Conditions {
   145  			if c.Type != apps.ReplicaSetReplicaFailure {
   146  				continue
   147  			}
   148  			conditions = append(conditions, util.ReplicaSetToDeploymentCondition(c))
   149  		}
   150  	}
   151  	return conditions
   152  }
   153  
   154  // used for unit testing
   155  var nowFn = func() time.Time { return time.Now() }
   156  
   157  // requeueStuckDeployment checks whether the provided deployment needs to be synced for a progress
   158  // check. It returns the time after the deployment will be requeued for the progress check, 0 if it
   159  // will be requeued now, or -1 if it does not need to be requeued.
   160  func (dc *DeploymentController) requeueStuckDeployment(ctx context.Context, d *apps.Deployment, newStatus apps.DeploymentStatus) time.Duration {
   161  	logger := klog.FromContext(ctx)
   162  	currentCond := util.GetDeploymentCondition(d.Status, apps.DeploymentProgressing)
   163  	// Can't estimate progress if there is no deadline in the spec or progressing condition in the current status.
   164  	if !util.HasProgressDeadline(d) || currentCond == nil {
   165  		return time.Duration(-1)
   166  	}
   167  	// No need to estimate progress if the rollout is complete or already timed out.
   168  	if util.DeploymentComplete(d, &newStatus) || currentCond.Reason == util.TimedOutReason {
   169  		return time.Duration(-1)
   170  	}
   171  	// If there is no sign of progress at this point then there is a high chance that the
   172  	// deployment is stuck. We should resync this deployment at some point in the future[1]
   173  	// and check whether it has timed out. We definitely need this, otherwise we depend on the
   174  	// controller resync interval. See https://github.com/kubernetes/kubernetes/issues/34458.
   175  	//
   176  	// [1] ProgressingCondition.LastUpdatedTime + progressDeadlineSeconds - time.Now()
   177  	//
   178  	// For example, if a Deployment updated its Progressing condition 3 minutes ago and has a
   179  	// deadline of 10 minutes, it would need to be resynced for a progress check after 7 minutes.
   180  	//
   181  	// lastUpdated: 			00:00:00
   182  	// now: 					00:03:00
   183  	// progressDeadlineSeconds: 600 (10 minutes)
   184  	//
   185  	// lastUpdated + progressDeadlineSeconds - now => 00:00:00 + 00:10:00 - 00:03:00 => 07:00
   186  	after := currentCond.LastUpdateTime.Time.Add(time.Duration(*d.Spec.ProgressDeadlineSeconds) * time.Second).Sub(nowFn())
   187  	// If the remaining time is less than a second, then requeue the deployment immediately.
   188  	// Make it ratelimited so we stay on the safe side, eventually the Deployment should
   189  	// transition either to a Complete or to a TimedOut condition.
   190  	if after < time.Second {
   191  		logger.V(4).Info("Queueing up deployment for a progress check now", "deployment", klog.KObj(d))
   192  		dc.enqueueRateLimited(d)
   193  		return time.Duration(0)
   194  	}
   195  	logger.V(4).Info("Queueing up deployment for a progress check", "deployment", klog.KObj(d), "queueAfter", int(after.Seconds()))
   196  	// Add a second to avoid milliseconds skew in AddAfter.
   197  	// See https://github.com/kubernetes/kubernetes/issues/39785#issuecomment-279959133 for more info.
   198  	dc.enqueueAfter(d, after+time.Second)
   199  	return after
   200  }
   201
View as plain text