patchorder.go

Documentation: edge-infra.dev/pkg/sds/patching/patchmanager

     1  package patchmanager
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"strings"
     7  	"time"
     8  
     9  	corev1 "k8s.io/api/core/v1"
    10  	"k8s.io/apimachinery/pkg/util/wait"
    11  	"sigs.k8s.io/controller-runtime/pkg/client"
    12  
    13  	"edge-infra.dev/pkg/k8s/runtime/conditions"
    14  	"edge-infra.dev/pkg/sds/patching/common"
    15  )
    16  
    17  const (
    18  	k8sAPICallRetrySleep   = 5 * time.Second
    19  	k8sAPICallRetryTimeout = 5 * time.Minute
    20  
    21  	upgradeRetrySleep = 2 * time.Minute
    22  	//No time out for waiting for upgrade
    23  )
    24  
    25  // Wait for control-plane to be upgraded. If node is control plane, reboot immediately.
    26  func (p *PatchManager) WaitForRebootAllowed() error {
    27  	if p.Ienpatch.Spec.DisablePatchOrdering {
    28  		p.Log.Info("disablePatchOrdering is True. Rebooting immediately")
    29  		return nil
    30  	}
    31  
    32  	controlPlane, err := p.getControlPlaneHostname()
    33  	if err != nil {
    34  		return fmt.Errorf("Could not find control plane: %w", err)
    35  	}
    36  
    37  	if p.HostName == controlPlane {
    38  		p.Log.Info("This node is control plane. Scheduling reboot")
    39  		return nil
    40  	}
    41  
    42  	p.Log.Info("This node is not the control plane. Waiting for it to be patched.")
    43  	if err = p.waitForControlPlaneUpgraded(controlPlane); err != nil {
    44  		return fmt.Errorf("Timeout waiting for control plane to upgrade: %w", err)
    45  	}
    46  	p.Log.Info("Control-Plane has been upgraded. Scheduling reboot")
    47  	return nil
    48  }
    49  
    50  // Wait for ienpatches CR for cp to change to ready
    51  func (p *PatchManager) waitForControlPlaneUpgraded(controlplane string) error {
    52  	var lastError error
    53  
    54  	err := wait.PollUntilContextCancel(p.Ctx, upgradeRetrySleep, true, func(ctx context.Context) (done bool, err error) {
    55  		lastError = p.K8sClient.Get(ctx, client.ObjectKey{Name: p.Ienpatch.Name, Namespace: p.Ienpatch.Namespace}, p.Ienpatch)
    56  		if lastError != nil {
    57  			p.Log.Error(lastError, "Waiting for control-plane upgrade, failed to contact api server. Retrying")
    58  			return false, nil
    59  		}
    60  		msg := conditions.GetMessage(p.Ienpatch, controlplane)
    61  		if conditions.IsTrue(p.Ienpatch, controlplane) && strings.HasSuffix(msg, p.TargetVer) && strings.HasPrefix(msg, "Success") {
    62  			return true, nil
    63  		}
    64  		p.Log.Info("Waiting for control-plane to upgrade to " + p.TargetVer)
    65  		return false, nil
    66  	})
    67  	if lastError != nil {
    68  		return lastError
    69  	}
    70  	return err
    71  }
    72  
    73  // Get hostname of control-plane
    74  // cluster must have only one control plane
    75  func (p *PatchManager) getControlPlaneHostname() (string, error) {
    76  	nodes := corev1.NodeList{}
    77  	var lastError error
    78  
    79  	err := wait.PollUntilContextTimeout(p.Ctx, k8sAPICallRetrySleep, k8sAPICallRetryTimeout, true, func(ctx context.Context) (done bool, err error) {
    80  		lastError = p.K8sClient.List(ctx, &nodes, client.MatchingLabels{common.ControlPlaneLabel: ""})
    81  		if lastError != nil {
    82  			p.Log.Error(lastError, "Attempting to find control-plane, failed to contact api server. Retrying")
    83  			return false, nil
    84  		}
    85  		return true, nil
    86  	})
    87  	if lastError != nil {
    88  		return "", lastError
    89  	}
    90  	if err != nil {
    91  		return "", err
    92  	}
    93  
    94  	if len(nodes.Items) == 1 {
    95  		return nodes.Items[0].Name, nil
    96  	}
    97  
    98  	return "", fmt.Errorf("Could not find control plane")
    99  }
   100
View as plain text