package edgeinjector import ( "context" "fmt" "time" "sigs.k8s.io/controller-runtime/pkg/controller" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" "edge-infra.dev/pkg/edge/clientutils" "edge-infra.dev/pkg/edge/controllers/envctl/pkg/nameutils" dsapi "edge-infra.dev/pkg/edge/datasync/apis/v1alpha1" "edge-infra.dev/pkg/edge/k8objectsutils/ownerref" "edge-infra.dev/pkg/lib/fog" nodemeta "edge-infra.dev/pkg/sds/ien/node" ) type PodSecretReconciler struct { client.Client Name string RequeueTime time.Duration PollingInterval time.Duration ReconcileConcurrency int } func (r *PodSecretReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&corev1.Pod{}, builder.WithPredicates(podPredicate())). WithOptions(controller.Options{MaxConcurrentReconciles: r.ReconcileConcurrency}). WithEventFilter(predicate.NewPredicateFuncs(func(obj client.Object) bool { if pod, ok := obj.(*corev1.Pod); ok { hasSecretLabel := SecretLabelValue(pod, CouchDBSecret) != "" || SecretLabelValue(pod, NodeSecret) != "" return hasSecretLabel && pod.Spec.NodeName != "" } return false })). Complete(r) } func podPredicate() predicate.Predicate { return predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { return e.ObjectNew.GetDeletionTimestamp().IsZero() }, CreateFunc: func(_ event.CreateEvent) bool { return true }, DeleteFunc: func(_ event.DeleteEvent) bool { return false }, } } func (r *PodSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := fog.FromContext(ctx) pod := &corev1.Pod{} if err := r.Client.Get(ctx, req.NamespacedName, pod); err != nil { return ctrl.Result{}, nil } log.Info("reconciling for pod secret") node := &corev1.Node{} if err := r.Client.Get(ctx, types.NamespacedName{Name: pod.Spec.NodeName}, node); err != nil { log.Error(err, "fail to get container's NodeName from pod, should have been filtered out", "nodeName", pod.Spec.NodeName) return ctrl.Result{}, nil } for _, secretType := range []SecretType{CouchDBSecret, NodeSecret} { secretName := SecretLabelValue(pod, secretType) if secretName == "" { continue } log := log.WithValues("nodeName", pod.Spec.NodeName, "secretName", secretName, "secretType", secretType) if err := r.SecretExists(ctx, secretType, pod.Namespace, secretName); err == nil { log.Info(fmt.Sprintf("secret: %s already exists", secretName)) continue } else if !errors.IsNotFound(err) { log.Error(err, "fail to check secret existence") return ctrl.Result{}, err } ctx = fog.IntoContext(ctx, log) var result ctrl.Result var err error switch secretType { case CouchDBSecret: result, err = r.CreatePodCouchDBSecret(ctx, node, pod) case NodeSecret: result, err = r.CreateNodeSecret(ctx, node, pod) default: err = fmt.Errorf("secret type not implemented: %s", secretType) } if err != nil { log.Error(err, "fail to create secret for pod") return result, err } if result.RequeueAfter > 0 { log.Info("re-trying to create node secret", "RequeueTime", result.RequeueAfter) return result, nil } } log.Info("Successfully reconcile secret: " + r.PollingInterval.String()) return ctrl.Result{RequeueAfter: r.PollingInterval}, nil } func (r *PodSecretReconciler) CreateNodeSecret(ctx context.Context, node *corev1.Node, pod *corev1.Pod) (ctrl.Result, error) { log := fog.FromContext(ctx) info, err := nameutils.GetNodeInfo(*node, nameutils.LaneNumberFullLength) if err != nil { log.Error(err, "fail to get node info, must be a dsds node") return ctrl.Result{RequeueAfter: r.RequeueTime}, nil } customLabels, err := GetNodeCustomLabels(node) if err != nil { log.Error(err, "fail to get node custom labels, must be a dsds node") return ctrl.Result{RequeueAfter: r.RequeueTime}, nil } secretName := SecretLabelValue(pod, NodeSecret) secret := BuildNodeSecret(pod, info, customLabels, secretName) if err := clientutils.CreateOrUpdateSecret(ctx, r, secret); err != nil { log.Error(err, "fail to create/update node env secret") return ctrl.Result{}, nil } log.Info("successfully created node info secret") return ctrl.Result{}, nil } func (r *PodSecretReconciler) CreatePodCouchDBSecret(ctx context.Context, node *corev1.Node, pod *corev1.Pod) (ctrl.Result, error) { log := fog.FromContext(ctx) isCP, err := nodemeta.IsControlPlaneNode(node) if err != nil { log.Error(err, "invalid node") return ctrl.Result{RequeueAfter: r.RequeueTime}, nil } serverName := dsapi.CouchDBServerName(node) secretName := SecretLabelValue(pod, CouchDBSecret) user, err := r.BuildCouchDBUser(ctx, pod, serverName, secretName, !isCP) if err != nil { log.Error(err, "fail to build CouchDBUser") // pod has invalid reference, must be re-created return ctrl.Result{}, nil } if err := clientutils.CreateOrUpdateCouchDBUser(ctx, r, user); err != nil { log.Error(err, "fail to create/update CouchDBUser", "CouchDBUser", user.Name) return ctrl.Result{RequeueAfter: r.RequeueTime}, nil } // note: couchctl manages the CouchDBUser secret lifecycle log.Info("successfully created couchdb user", "CouchDBUser", user.Name) return ctrl.Result{}, nil } func (r *PodSecretReconciler) BuildCouchDBUser(ctx context.Context, pod *corev1.Pod, serverName, secretName string, isTouchPoint bool) (*dsapi.CouchDBUser, error) { log := fog.FromContext(ctx) ownerRef, err := ownerref.GetOwnerRef(ctx, r, pod) if err != nil { log.Error(err, "fail to get valid owner reference for pod", "Valid OwnerRefs", "Deployment,DaemonSet,StatefulSet") // TODO ignore error return nil, err } userOwnerRef := *ownerRef f := false userOwnerRef.Controller = &f username := ownerref.ResourceName(serverName, userOwnerRef, pod.Namespace, pod.Spec.NodeName, isTouchPoint) user := dsapi.NewCouchDBUser(dsapi.UserCredentials, username, serverName, CouchDBUserRole(pod)) user.Namespace = pod.Namespace // Secret to be created by couchctl user.Spec.User.Secret.Namespace = pod.Namespace user.Spec.User.Secret.Name = secretName user.ObjectMeta.OwnerReferences = []metav1.OwnerReference{userOwnerRef} return user, nil } func (r *PodSecretReconciler) SecretExists(ctx context.Context, st SecretType, namespace, name string) error { s := &corev1.Secret{} err := r.Client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, s) if err != nil { return err } if st == NodeSecret { return checkNodeSecret(s) } return nil }