provisioner.go

Documentation: edge-infra.dev/pkg/sds/ien/k8s/controllers/pxe

     1  package pxe
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"os"
     8  	"strings"
     9  
    10  	corev1 "k8s.io/api/core/v1"
    11  	kerrors "k8s.io/apimachinery/pkg/api/errors"
    12  	"k8s.io/apimachinery/pkg/types"
    13  	"k8s.io/client-go/dynamic"
    14  	"sigs.k8s.io/cli-utils/pkg/kstatus/watcher"
    15  	ctrl "sigs.k8s.io/controller-runtime"
    16  	"sigs.k8s.io/controller-runtime/pkg/builder"
    17  	"sigs.k8s.io/controller-runtime/pkg/client"
    18  	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
    19  	"sigs.k8s.io/controller-runtime/pkg/event"
    20  	"sigs.k8s.io/controller-runtime/pkg/handler"
    21  	"sigs.k8s.io/controller-runtime/pkg/predicate"
    22  	"sigs.k8s.io/controller-runtime/pkg/reconcile"
    23  
    24  	"edge-infra.dev/pkg/edge/info"
    25  	"edge-infra.dev/pkg/edge/k8objectsutils"
    26  	"edge-infra.dev/pkg/k8s/runtime/patch"
    27  	"edge-infra.dev/pkg/k8s/runtime/sap"
    28  	"edge-infra.dev/pkg/lib/fog"
    29  	"edge-infra.dev/pkg/sds/ien/bootoptions"
    30  	v1ienode "edge-infra.dev/pkg/sds/ien/k8s/apis/v1"
    31  	v1pxe "edge-infra.dev/pkg/sds/ien/k8s/controllers/pxe/apis/v1"
    32  	"edge-infra.dev/pkg/sds/ien/k8s/controllers/pxe/common"
    33  	"edge-infra.dev/pkg/sds/ien/k8s/controllers/pxe/dnsmasq"
    34  )
    35  
    36  const (
    37  	ActivationCodeConfigPrefix = "activation-code"
    38  	provisionerPXEName         = "provisioner"
    39  )
    40  
    41  // Provisioner acts as a controller that can be used to create the resources
    42  // required for a node to be PXE booted
    43  type Provisioner struct {
    44  	Reconciler
    45  }
    46  
    47  // NewProvisioner returns a new Provisioner using the provided client
    48  func NewProvisioner(mgr ctrl.Manager, cli client.Client) (*Provisioner, error) {
    49  	name := provisionerPXEName
    50  
    51  	d, err := dynamic.NewForConfig(mgr.GetConfig())
    52  	if err != nil {
    53  		return nil, err
    54  	}
    55  
    56  	manager := sap.NewResourceManager(
    57  		cli,
    58  		watcher.NewDefaultStatusWatcher(d, mgr.GetRESTMapper()),
    59  		sap.Owner{Field: name},
    60  	)
    61  
    62  	return &Provisioner{
    63  		Reconciler: Reconciler{
    64  			name,
    65  			cli,
    66  			manager,
    67  		},
    68  	}, nil
    69  }
    70  
    71  // SetupWithManager sets up the provisioner controller with the provided manager
    72  func (p *Provisioner) SetupWithManager(mgr ctrl.Manager) error {
    73  	hostname := os.Getenv("NODE_NAME")
    74  	if hostname == "" {
    75  		return errors.New("NODE_NAME environment variable is not set")
    76  	}
    77  
    78  	return ctrl.NewControllerManagedBy(mgr).
    79  		For(
    80  			&v1pxe.PXE{},
    81  			builder.WithPredicates(predicate.GenerationChangedPredicate{}, p.isProvisioner(), p.ignoreDelete(), predicate.Or(p.isDeleteRequested(), p.isSuspendChanged())),
    82  		).
    83  		Watches(
    84  			&v1ienode.IENode{},
    85  			handler.EnqueueRequestsFromMapFunc(p.createReconcileRequest),
    86  			builder.WithPredicates(predicate.GenerationChangedPredicate{}, p.isNotNode(hostname)),
    87  		).
    88  		Watches( // react to creation and deletion of nodes as may need to either pxe boot or clean up
    89  			&corev1.Node{},
    90  			handler.EnqueueRequestsFromMapFunc(p.createReconcileRequest),
    91  			builder.WithPredicates(p.ignoreUpdate(), p.isNotNode(hostname)),
    92  		).
    93  		Watches( // react to changes to boot-options ConfigMap
    94  			&corev1.ConfigMap{},
    95  			handler.EnqueueRequestsFromMapFunc(p.createReconcileRequest),
    96  			builder.WithPredicates(p.isBootOptionsConfigMap(), p.ignoreDelete()),
    97  		).
    98  		Watches( // react to creation of Secrets with name prefixed with "activation-code"
    99  			&corev1.Secret{},
   100  			handler.EnqueueRequestsFromMapFunc(p.createReconcileRequest),
   101  			builder.WithPredicates(p.isActivationCodeSecret(), p.ignoreDelete()),
   102  		).Complete(p)
   103  }
   104  
   105  // isProvisioner returns a predicate func that will return true if the object
   106  // has the provisioner pxe name, otherwise it will return false
   107  func (p *Provisioner) isProvisioner() predicate.Funcs {
   108  	return predicate.NewPredicateFuncs(func(obj client.Object) bool {
   109  		return obj.GetName() == provisionerPXEName
   110  	})
   111  }
   112  
   113  // isDeleteRequested returns a predicate func that will return true if the
   114  // object has been requested to be deleted, otherwise it will return false
   115  func (p *Provisioner) isDeleteRequested() predicate.Funcs {
   116  	return predicate.NewPredicateFuncs(func(obj client.Object) bool {
   117  		return !obj.GetDeletionTimestamp().IsZero()
   118  	})
   119  }
   120  
   121  // isSuspendChanged returns a predicate func that will return true if the object
   122  // has been updated and in that update the suspend option was changed, otherwise
   123  // will return false
   124  func (p *Provisioner) isSuspendChanged() predicate.Funcs {
   125  	return predicate.Funcs{
   126  		CreateFunc: func(_ event.CreateEvent) bool {
   127  			return false
   128  		},
   129  		UpdateFunc: func(e event.UpdateEvent) bool {
   130  			pxeOld := e.ObjectOld.(*v1pxe.PXE)
   131  			pxeNew := e.ObjectNew.(*v1pxe.PXE)
   132  
   133  			return pxeOld.Spec.Suspend != pxeNew.Spec.Suspend
   134  		},
   135  		DeleteFunc: func(_ event.DeleteEvent) bool {
   136  			return false
   137  		},
   138  	}
   139  }
   140  
   141  // isNotNode returns a predicate func that returns false if the provided event
   142  // object has the same name as the provided nodeName, otherwise returns true
   143  func (p *Provisioner) isNotNode(nodeName string) predicate.Funcs {
   144  	return predicate.NewPredicateFuncs(func(obj client.Object) bool {
   145  		return obj.GetName() != nodeName
   146  	})
   147  }
   148  
   149  // isBootOptionsConfigMap returns a predicate func that will return true if the
   150  // provided object is the boot options configmap, otherwise it will return false
   151  func (p *Provisioner) isBootOptionsConfigMap() predicate.Funcs {
   152  	return predicate.NewPredicateFuncs(func(obj client.Object) bool {
   153  		return bootoptions.IsBootOptionsConfigMap(obj)
   154  	})
   155  }
   156  
   157  // isActivationCodeSecret returns a predicate func that will return true if the
   158  // provided object is an activation code secret, otherwise it will return false
   159  func (p *Provisioner) isActivationCodeSecret() predicate.Funcs {
   160  	return predicate.NewPredicateFuncs(func(obj client.Object) bool {
   161  		return obj.GetNamespace() == common.PXENamespace &&
   162  			strings.HasPrefix(obj.GetName(), ActivationCodeConfigPrefix)
   163  	})
   164  }
   165  
   166  // ignoreUpdate returns a predicate func that ignores object update events
   167  func (p *Provisioner) ignoreUpdate() predicate.Funcs {
   168  	return predicate.Funcs{
   169  		UpdateFunc: func(_ event.UpdateEvent) bool {
   170  			return false
   171  		},
   172  	}
   173  }
   174  
   175  // ignoreDelete returns a predicate func that ignores object delete events
   176  func (p *Provisioner) ignoreDelete() predicate.Funcs {
   177  	return predicate.Funcs{
   178  		DeleteFunc: func(_ event.DeleteEvent) bool {
   179  			return false
   180  		},
   181  	}
   182  }
   183  
   184  // createReconcileRequest returns a reconcile request for provisioner pxe
   185  // instance
   186  func (p *Provisioner) createReconcileRequest(_ context.Context, _ client.Object) []reconcile.Request {
   187  	return []reconcile.Request{
   188  		{
   189  			NamespacedName: client.ObjectKey{
   190  				Name: provisionerPXEName,
   191  			},
   192  		},
   193  	}
   194  }
   195  
   196  // Reconcile on the provisioner PXE instance. During reconciliation, it will be
   197  // determined which nodes could currently be eligible to be PXE booted. Any
   198  // nodes that are found to be eligible will have the resources they require for
   199  // PXE booting created, all others will have theirs deleted (if they had any
   200  // provisioned already). Each eligible node will have a dnsmasq-options,
   201  // dhcp-hosts and a dhcp-options resource created, as well as an entry into the
   202  // ipxe-files configmap
   203  func (p *Provisioner) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, recErr error) {
   204  	log := fog.FromContext(ctx).WithValues("reconciler", p.name)
   205  	ctx = fog.IntoContext(ctx, log)
   206  
   207  	log.Info("reconciling")
   208  
   209  	pxe := &v1pxe.PXE{}
   210  	if err := p.client.Get(ctx, req.NamespacedName, pxe); err != nil {
   211  		return ctrl.Result{}, client.IgnoreNotFound(err)
   212  	}
   213  
   214  	// if provisioner pxe is disabled, do nothing
   215  	if pxe.Spec.Suspend {
   216  		log.Info("reconciliation suspended")
   217  
   218  		return ctrl.Result{}, nil
   219  	}
   220  
   221  	pxeMgr := newPXEManager(p.manager, pxe)
   222  
   223  	options, err := bootoptions.FromClient(ctx, p.client)
   224  	if err != nil {
   225  		return ctrl.Result{}, err
   226  	}
   227  
   228  	patcher := patch.NewSerialPatcher(pxe, p.client)
   229  	defer func() {
   230  		res, recErr = p.summarize(ctx, patcher, pxe, recErr)
   231  	}()
   232  
   233  	// if PXE booting is disabled then clean up the PXE booting resources for
   234  	// this node
   235  	if !options.PXEEnabled {
   236  		log.Info("PXE booting is disabled - cleaning up PXE resources for all nodes")
   237  
   238  		return ctrl.Result{}, pxeMgr.apply(ctx)
   239  	}
   240  
   241  	// is pxe resource has been scheduled for deletion
   242  	if !pxe.GetDeletionTimestamp().IsZero() {
   243  		log.Info("PXE resource scheduled for deletion - cleaning up PXE resources for all nodes")
   244  
   245  		controllerutil.RemoveFinalizer(pxe, v1pxe.Finalizer)
   246  		return ctrl.Result{}, pxeMgr.apply(ctx)
   247  	}
   248  
   249  	ienodeList := &v1ienode.IENodeList{}
   250  	if err := p.client.List(ctx, ienodeList); err != nil {
   251  		return ctrl.Result{}, err
   252  	}
   253  
   254  	var params []dnsmasq.NodeDNSMasqParams
   255  	var manifests [][]byte
   256  	for i := range ienodeList.Items {
   257  		// ignore the node that the controller is running on
   258  		if ienodeList.Items[i].Name == os.Getenv("NODE_NAME") {
   259  			continue
   260  		}
   261  
   262  		// get the DNSMasq options, DHCP options and DHCP hosts resources for
   263  		// the node, as well as the params needed to fill out the ipxe field of
   264  		// the ipxe-files configmap
   265  		p, m, err := p.nodeDNSMasqConfig(ctx, p.client, &ienodeList.Items[i])
   266  		if err != nil {
   267  			return ctrl.Result{}, err
   268  		}
   269  
   270  		if p != nil {
   271  			params = append(params, *p)
   272  		}
   273  		manifests = append(manifests, m...)
   274  	}
   275  
   276  	// use the params retrieved above to template the ipxe-files configmap
   277  	ipxeConfigMap, err := dnsmasq.NodeIPXEConfigMap(params)
   278  	if err != nil {
   279  		return ctrl.Result{}, err
   280  	}
   281  	manifests = append(manifests, ipxeConfigMap)
   282  
   283  	log.Info("updating PXE boot config for nodes")
   284  	return ctrl.Result{}, pxeMgr.apply(ctx, manifests...)
   285  }
   286  
   287  // nodeDNSMasqConfig will check to see if the provided IENode is eligible to be
   288  // PXE booted. If it is then it will get the activation code and API endpoint
   289  // required for the node and use them (and the IENode resource itself) to
   290  // generate the parameters required to generate the manifests
   291  func (p *Provisioner) nodeDNSMasqConfig(ctx context.Context, cli client.Client, ienode *v1ienode.IENode) (params *dnsmasq.NodeDNSMasqParams, manifests [][]byte, err error) {
   292  	eligible, err := p.checkEligibility(ctx, ienode)
   293  	if err != nil {
   294  		return nil, nil, err
   295  	}
   296  	// if IENode is not eligible for PXE booting, do not generate PXE booting
   297  	// resources
   298  	if !eligible {
   299  		return nil, nil, nil
   300  	}
   301  
   302  	options, err := bootoptions.FromClient(ctx, cli)
   303  	if err != nil {
   304  		return nil, nil, err
   305  	}
   306  
   307  	// gather relevant IENode data and boot-options CM
   308  	activationCode, apiEndpoint, err := pxeData(ctx, cli, ienode, options.ACRelay)
   309  	if err != nil {
   310  		return nil, nil, err
   311  	}
   312  	// if we should have been able to get the activation code but did not
   313  	if options.ACRelay && activationCode == "" {
   314  		return nil, nil, nil
   315  	}
   316  
   317  	params, err = dnsmasq.RenderNodeDNSMasqParams(ienode, activationCode, apiEndpoint)
   318  	if err != nil {
   319  		return nil, nil, err
   320  	}
   321  
   322  	manifests, err = dnsmasq.NodeDNSMasqManifests(*params)
   323  	if err != nil {
   324  		return nil, nil, err
   325  	}
   326  
   327  	return params, manifests, nil
   328  }
   329  
   330  // checkEligibility returns true if the provided IENode is eligible for PXE
   331  // booting, otherwise returns false
   332  func (p *Provisioner) checkEligibility(ctx context.Context, ienode *v1ienode.IENode) (bool, error) {
   333  	log := fog.FromContext(ctx).WithValues("node", ienode.Name)
   334  
   335  	err := p.client.Get(ctx, client.ObjectKeyFromObject(ienode), &corev1.Node{})
   336  	if client.IgnoreNotFound(err) != nil {
   337  		return false, err
   338  	}
   339  	// if node already exists in the cluster then it has already been installed
   340  	if err == nil {
   341  		log.Info("node already exists in the cluster")
   342  		return false, nil
   343  	}
   344  
   345  	if ienode.Spec.Network[0].DHCP4 {
   346  		log.Info("DHCP is enabled - node ineligible to be PXE booted", "pxeaudit", "")
   347  		return false, nil
   348  	}
   349  
   350  	if isValid, err := ienode.Spec.IsValidStaticIPConfiguration(); !isValid {
   351  		log.Error(err, "invalid static IP configuration")
   352  		return false, nil
   353  	}
   354  
   355  	return true, nil
   356  }
   357  
   358  // pxeData returns the activation code and API endpoint required when installing
   359  // the provided IENode
   360  func pxeData(ctx context.Context, cli client.Client, ienode *v1ienode.IENode, acRelay bool) (code string, endpoint string, err error) {
   361  	log := fog.FromContext(ctx)
   362  
   363  	// if we should be able to automatically retrieve the activation code, get it
   364  	if acRelay {
   365  		code, err = activationCode(ctx, ienode, cli)
   366  		if kerrors.IsNotFound(err) {
   367  			log.Info("no activation code exists for this node - pxe config removed or is not yet ready")
   368  			return "", "", nil
   369  		}
   370  
   371  		if err != nil {
   372  			return "", "", fmt.Errorf("error getting activation code secret: %v", err)
   373  		}
   374  	}
   375  
   376  	endpoint, err = apiEndpoint(ctx, cli)
   377  	if err != nil {
   378  		return "", "", err
   379  	}
   380  
   381  	return code, endpoint, nil
   382  }
   383  
   384  // activationCode retrieves the activation code from the activation code secret
   385  // data
   386  func activationCode(ctx context.Context, ienode *v1ienode.IENode, cli client.Client) (string, error) {
   387  	secret := &corev1.Secret{}
   388  	if err := activationCodeSecret(ctx, ienode, cli, secret); err != nil {
   389  		return "", err
   390  	}
   391  
   392  	return string(secret.Data["activation"]), nil
   393  }
   394  
   395  // activationCodeSecret returns the activation code secret for the provided
   396  // IENode
   397  func activationCodeSecret(ctx context.Context, ienode *v1ienode.IENode, cli client.Client, secret *corev1.Secret) error {
   398  	terminalID := ienode.ObjectMeta.Labels["node.ncr.com/terminal-id"]
   399  	if terminalID == "" {
   400  		return fmt.Errorf("node has no terminalID label, can't load activation code secret")
   401  	}
   402  
   403  	namespacedname := types.NamespacedName{
   404  		Namespace: common.PXENamespace,
   405  		Name:      k8objectsutils.NameWithPrefix(ActivationCodeConfigPrefix, terminalID),
   406  	}
   407  
   408  	return cli.Get(ctx, namespacedname, secret)
   409  }
   410  
   411  // apiEndpoint returns the API endpoint to be used for node installations
   412  func apiEndpoint(ctx context.Context, cli client.Client) (string, error) {
   413  	cm := &corev1.ConfigMap{}
   414  	key := client.ObjectKey{
   415  		Name:      info.EdgeConfigMapName,
   416  		Namespace: common.PXENamespace,
   417  	}
   418  
   419  	if err := cli.Get(ctx, key, cm); err != nil {
   420  		return "", err
   421  	}
   422  
   423  	return info.FromConfigMap(cm).EdgeAPIEndpoint, nil
   424  }
   425
View as plain text