package pxe import ( "context" "errors" "fmt" "os" "strings" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" "sigs.k8s.io/cli-utils/pkg/kstatus/watcher" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "edge-infra.dev/pkg/edge/info" "edge-infra.dev/pkg/edge/k8objectsutils" "edge-infra.dev/pkg/k8s/runtime/patch" "edge-infra.dev/pkg/k8s/runtime/sap" "edge-infra.dev/pkg/lib/fog" "edge-infra.dev/pkg/sds/ien/bootoptions" v1ienode "edge-infra.dev/pkg/sds/ien/k8s/apis/v1" v1pxe "edge-infra.dev/pkg/sds/ien/k8s/controllers/pxe/apis/v1" "edge-infra.dev/pkg/sds/ien/k8s/controllers/pxe/common" "edge-infra.dev/pkg/sds/ien/k8s/controllers/pxe/dnsmasq" ) const ( ActivationCodeConfigPrefix = "activation-code" provisionerPXEName = "provisioner" ) // Provisioner acts as a controller that can be used to create the resources // required for a node to be PXE booted type Provisioner struct { Reconciler } // NewProvisioner returns a new Provisioner using the provided client func NewProvisioner(mgr ctrl.Manager, cli client.Client) (*Provisioner, error) { name := provisionerPXEName d, err := dynamic.NewForConfig(mgr.GetConfig()) if err != nil { return nil, err } manager := sap.NewResourceManager( cli, watcher.NewDefaultStatusWatcher(d, mgr.GetRESTMapper()), sap.Owner{Field: name}, ) return &Provisioner{ Reconciler: Reconciler{ name, cli, manager, }, }, nil } // SetupWithManager sets up the provisioner controller with the provided manager func (p *Provisioner) SetupWithManager(mgr ctrl.Manager) error { hostname := os.Getenv("NODE_NAME") if hostname == "" { return errors.New("NODE_NAME environment variable is not set") } return ctrl.NewControllerManagedBy(mgr). For( &v1pxe.PXE{}, builder.WithPredicates(predicate.GenerationChangedPredicate{}, p.isProvisioner(), p.ignoreDelete(), predicate.Or(p.isDeleteRequested(), p.isSuspendChanged())), ). Watches( &v1ienode.IENode{}, handler.EnqueueRequestsFromMapFunc(p.createReconcileRequest), builder.WithPredicates(predicate.GenerationChangedPredicate{}, p.isNotNode(hostname)), ). Watches( // react to creation and deletion of nodes as may need to either pxe boot or clean up &corev1.Node{}, handler.EnqueueRequestsFromMapFunc(p.createReconcileRequest), builder.WithPredicates(p.ignoreUpdate(), p.isNotNode(hostname)), ). Watches( // react to changes to boot-options ConfigMap &corev1.ConfigMap{}, handler.EnqueueRequestsFromMapFunc(p.createReconcileRequest), builder.WithPredicates(p.isBootOptionsConfigMap(), p.ignoreDelete()), ). Watches( // react to creation of Secrets with name prefixed with "activation-code" &corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(p.createReconcileRequest), builder.WithPredicates(p.isActivationCodeSecret(), p.ignoreDelete()), ).Complete(p) } // isProvisioner returns a predicate func that will return true if the object // has the provisioner pxe name, otherwise it will return false func (p *Provisioner) isProvisioner() predicate.Funcs { return predicate.NewPredicateFuncs(func(obj client.Object) bool { return obj.GetName() == provisionerPXEName }) } // isDeleteRequested returns a predicate func that will return true if the // object has been requested to be deleted, otherwise it will return false func (p *Provisioner) isDeleteRequested() predicate.Funcs { return predicate.NewPredicateFuncs(func(obj client.Object) bool { return !obj.GetDeletionTimestamp().IsZero() }) } // isSuspendChanged returns a predicate func that will return true if the object // has been updated and in that update the suspend option was changed, otherwise // will return false func (p *Provisioner) isSuspendChanged() predicate.Funcs { return predicate.Funcs{ CreateFunc: func(_ event.CreateEvent) bool { return false }, UpdateFunc: func(e event.UpdateEvent) bool { pxeOld := e.ObjectOld.(*v1pxe.PXE) pxeNew := e.ObjectNew.(*v1pxe.PXE) return pxeOld.Spec.Suspend != pxeNew.Spec.Suspend }, DeleteFunc: func(_ event.DeleteEvent) bool { return false }, } } // isNotNode returns a predicate func that returns false if the provided event // object has the same name as the provided nodeName, otherwise returns true func (p *Provisioner) isNotNode(nodeName string) predicate.Funcs { return predicate.NewPredicateFuncs(func(obj client.Object) bool { return obj.GetName() != nodeName }) } // isBootOptionsConfigMap returns a predicate func that will return true if the // provided object is the boot options configmap, otherwise it will return false func (p *Provisioner) isBootOptionsConfigMap() predicate.Funcs { return predicate.NewPredicateFuncs(func(obj client.Object) bool { return bootoptions.IsBootOptionsConfigMap(obj) }) } // isActivationCodeSecret returns a predicate func that will return true if the // provided object is an activation code secret, otherwise it will return false func (p *Provisioner) isActivationCodeSecret() predicate.Funcs { return predicate.NewPredicateFuncs(func(obj client.Object) bool { return obj.GetNamespace() == common.PXENamespace && strings.HasPrefix(obj.GetName(), ActivationCodeConfigPrefix) }) } // ignoreUpdate returns a predicate func that ignores object update events func (p *Provisioner) ignoreUpdate() predicate.Funcs { return predicate.Funcs{ UpdateFunc: func(_ event.UpdateEvent) bool { return false }, } } // ignoreDelete returns a predicate func that ignores object delete events func (p *Provisioner) ignoreDelete() predicate.Funcs { return predicate.Funcs{ DeleteFunc: func(_ event.DeleteEvent) bool { return false }, } } // createReconcileRequest returns a reconcile request for provisioner pxe // instance func (p *Provisioner) createReconcileRequest(_ context.Context, _ client.Object) []reconcile.Request { return []reconcile.Request{ { NamespacedName: client.ObjectKey{ Name: provisionerPXEName, }, }, } } // Reconcile on the provisioner PXE instance. During reconciliation, it will be // determined which nodes could currently be eligible to be PXE booted. Any // nodes that are found to be eligible will have the resources they require for // PXE booting created, all others will have theirs deleted (if they had any // provisioned already). Each eligible node will have a dnsmasq-options, // dhcp-hosts and a dhcp-options resource created, as well as an entry into the // ipxe-files configmap func (p *Provisioner) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, recErr error) { log := fog.FromContext(ctx).WithValues("reconciler", p.name) ctx = fog.IntoContext(ctx, log) log.Info("reconciling") pxe := &v1pxe.PXE{} if err := p.client.Get(ctx, req.NamespacedName, pxe); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } // if provisioner pxe is disabled, do nothing if pxe.Spec.Suspend { log.Info("reconciliation suspended") return ctrl.Result{}, nil } pxeMgr := newPXEManager(p.manager, pxe) options, err := bootoptions.FromClient(ctx, p.client) if err != nil { return ctrl.Result{}, err } patcher := patch.NewSerialPatcher(pxe, p.client) defer func() { res, recErr = p.summarize(ctx, patcher, pxe, recErr) }() // if PXE booting is disabled then clean up the PXE booting resources for // this node if !options.PXEEnabled { log.Info("PXE booting is disabled - cleaning up PXE resources for all nodes") return ctrl.Result{}, pxeMgr.apply(ctx) } // is pxe resource has been scheduled for deletion if !pxe.GetDeletionTimestamp().IsZero() { log.Info("PXE resource scheduled for deletion - cleaning up PXE resources for all nodes") controllerutil.RemoveFinalizer(pxe, v1pxe.Finalizer) return ctrl.Result{}, pxeMgr.apply(ctx) } ienodeList := &v1ienode.IENodeList{} if err := p.client.List(ctx, ienodeList); err != nil { return ctrl.Result{}, err } var params []dnsmasq.NodeDNSMasqParams var manifests [][]byte for i := range ienodeList.Items { // ignore the node that the controller is running on if ienodeList.Items[i].Name == os.Getenv("NODE_NAME") { continue } // get the DNSMasq options, DHCP options and DHCP hosts resources for // the node, as well as the params needed to fill out the ipxe field of // the ipxe-files configmap p, m, err := p.nodeDNSMasqConfig(ctx, p.client, &ienodeList.Items[i]) if err != nil { return ctrl.Result{}, err } if p != nil { params = append(params, *p) } manifests = append(manifests, m...) } // use the params retrieved above to template the ipxe-files configmap ipxeConfigMap, err := dnsmasq.NodeIPXEConfigMap(params) if err != nil { return ctrl.Result{}, err } manifests = append(manifests, ipxeConfigMap) log.Info("updating PXE boot config for nodes") return ctrl.Result{}, pxeMgr.apply(ctx, manifests...) } // nodeDNSMasqConfig will check to see if the provided IENode is eligible to be // PXE booted. If it is then it will get the activation code and API endpoint // required for the node and use them (and the IENode resource itself) to // generate the parameters required to generate the manifests func (p *Provisioner) nodeDNSMasqConfig(ctx context.Context, cli client.Client, ienode *v1ienode.IENode) (params *dnsmasq.NodeDNSMasqParams, manifests [][]byte, err error) { eligible, err := p.checkEligibility(ctx, ienode) if err != nil { return nil, nil, err } // if IENode is not eligible for PXE booting, do not generate PXE booting // resources if !eligible { return nil, nil, nil } options, err := bootoptions.FromClient(ctx, cli) if err != nil { return nil, nil, err } // gather relevant IENode data and boot-options CM activationCode, apiEndpoint, err := pxeData(ctx, cli, ienode, options.ACRelay) if err != nil { return nil, nil, err } // if we should have been able to get the activation code but did not if options.ACRelay && activationCode == "" { return nil, nil, nil } params, err = dnsmasq.RenderNodeDNSMasqParams(ienode, activationCode, apiEndpoint) if err != nil { return nil, nil, err } manifests, err = dnsmasq.NodeDNSMasqManifests(*params) if err != nil { return nil, nil, err } return params, manifests, nil } // checkEligibility returns true if the provided IENode is eligible for PXE // booting, otherwise returns false func (p *Provisioner) checkEligibility(ctx context.Context, ienode *v1ienode.IENode) (bool, error) { log := fog.FromContext(ctx).WithValues("node", ienode.Name) err := p.client.Get(ctx, client.ObjectKeyFromObject(ienode), &corev1.Node{}) if client.IgnoreNotFound(err) != nil { return false, err } // if node already exists in the cluster then it has already been installed if err == nil { log.Info("node already exists in the cluster") return false, nil } if ienode.Spec.Network[0].DHCP4 { log.Info("DHCP is enabled - node ineligible to be PXE booted", "pxeaudit", "") return false, nil } if isValid, err := ienode.Spec.IsValidStaticIPConfiguration(); !isValid { log.Error(err, "invalid static IP configuration") return false, nil } return true, nil } // pxeData returns the activation code and API endpoint required when installing // the provided IENode func pxeData(ctx context.Context, cli client.Client, ienode *v1ienode.IENode, acRelay bool) (code string, endpoint string, err error) { log := fog.FromContext(ctx) // if we should be able to automatically retrieve the activation code, get it if acRelay { code, err = activationCode(ctx, ienode, cli) if kerrors.IsNotFound(err) { log.Info("no activation code exists for this node - pxe config removed or is not yet ready") return "", "", nil } if err != nil { return "", "", fmt.Errorf("error getting activation code secret: %v", err) } } endpoint, err = apiEndpoint(ctx, cli) if err != nil { return "", "", err } return code, endpoint, nil } // activationCode retrieves the activation code from the activation code secret // data func activationCode(ctx context.Context, ienode *v1ienode.IENode, cli client.Client) (string, error) { secret := &corev1.Secret{} if err := activationCodeSecret(ctx, ienode, cli, secret); err != nil { return "", err } return string(secret.Data["activation"]), nil } // activationCodeSecret returns the activation code secret for the provided // IENode func activationCodeSecret(ctx context.Context, ienode *v1ienode.IENode, cli client.Client, secret *corev1.Secret) error { terminalID := ienode.ObjectMeta.Labels["node.ncr.com/terminal-id"] if terminalID == "" { return fmt.Errorf("node has no terminalID label, can't load activation code secret") } namespacedname := types.NamespacedName{ Namespace: common.PXENamespace, Name: k8objectsutils.NameWithPrefix(ActivationCodeConfigPrefix, terminalID), } return cli.Get(ctx, namespacedname, secret) } // apiEndpoint returns the API endpoint to be used for node installations func apiEndpoint(ctx context.Context, cli client.Client) (string, error) { cm := &corev1.ConfigMap{} key := client.ObjectKey{ Name: info.EdgeConfigMapName, Namespace: common.PXENamespace, } if err := cli.Get(ctx, key, cm); err != nil { return "", err } return info.FromConfigMap(cm).EdgeAPIEndpoint, nil }