package bannerctl import ( "context" "database/sql" "encoding/json" "errors" "fmt" "time" "github.com/google/uuid" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" kms "cloud.google.com/go/kms/apiv1" "cloud.google.com/go/kms/apiv1/kmspb" "github.com/go-logr/logr" "github.com/golang-jwt/jwt" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "edge-infra.dev/pkg/edge/api/services/channels" bannerAPI "edge-infra.dev/pkg/edge/apis/banner/v1alpha1" "edge-infra.dev/pkg/edge/edgeencrypt" "edge-infra.dev/pkg/k8s/runtime/controller/reconcile" "edge-infra.dev/pkg/k8s/runtime/controller/reconcile/recerr" ) type EncryptionInfraReconciler struct { client.Client Log logr.Logger Conditions reconcile.Conditions ForemanProjectID string GCPRegion string SecretManager secretManager KmsClient *kms.KeyManagementClient kmsKey edgeencrypt.KmsKey SigningMethod jwt.SigningMethod ChannelService channels.Service IntervalTime time.Duration RequeueTime time.Duration ResourceTimeout time.Duration } func (r *EncryptionInfraReconciler) SetupWithManager(mgr ctrl.Manager) error { r.kmsKey = edgeencrypt.KmsKey{ProjectID: r.ForemanProjectID, Location: r.GCPRegion} return ctrl.NewControllerManagedBy(mgr). For(&bannerAPI.Banner{}). Complete(r) } func (r *EncryptionInfraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, recErr error) { //nolint:dupl banner := &bannerAPI.Banner{} if err := r.Get(ctx, req.NamespacedName, banner); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } log := ctrl.Log.WithValues("banner", banner.Name) // Get the list of channels for the banner bcs, err := getBannerChannels(ctx, r.ChannelService, banner.Name) if err != nil { return ctrl.Result{RequeueAfter: r.IntervalTime}, nil } if len(bcs) == 0 { return ctrl.Result{RequeueAfter: r.RequeueTime}, nil } recErr = r.reconcile(ctx, banner, bcs) if recErr != nil { log.Error(recErr, "failed to reconcile") return ctrl.Result{RequeueAfter: r.IntervalTime}, nil } return ctrl.Result{RequeueAfter: r.RequeueTime}, nil } func (r *EncryptionInfraReconciler) reconcile(ctx context.Context, banner *bannerAPI.Banner, channels []channels.BannerChannel) recerr.Error { secretClient, err := r.SecretManager.NewWithOptions(ctx, banner.Spec.GCP.ProjectID) if err != nil { return recerr.New(err, bannerAPI.PlatformSecretsCreationFailedReason) } //defer secretClient.Close() // create key ring per bearer err = r.createKeyRing(ctx, banner.Name) if err != nil { return recerr.New(err, bannerAPI.PlatformSecretsCreationFailedReason) } // create encryption jwt signing key per banner err = r.createKMSSigningKey(ctx, secretClient, banner, edgeencrypt.EncryptionJWTSecret, edgeencrypt.EncryptionJWTSecretManager, map[string]string{"banner": banner.Name}) if err != nil { return recerr.New(err, bannerAPI.PlatformSecretsCreationFailedReason) } bannerEdgeID, err := uuid.Parse(banner.Name) if err != nil { return recerr.New(err, bannerAPI.PlatformSecretsCreationFailedReason) } for _, channel := range channels { // create encryption key for data per channel per banner err = r.createKMSEncryptionKey(ctx, secretClient, banner, fmt.Sprintf(edgeencrypt.EncryptionSecret, channel.ID), fmt.Sprintf(edgeencrypt.EncryptionSecretManager, channel.ID), map[string]string{"channel": channel.Name}, ) if err != nil { return recerr.New(err, bannerAPI.PlatformSecretsCreationFailedReason) } // save channel key version to db err = r.CreateChannelKeyVersion(ctx, bannerEdgeID, channel.ID, banner.Spec.GCP.ProjectID, fmt.Sprintf(edgeencrypt.EncryptionSecretManager, channel.ID)) if err != nil { return recerr.New(err, bannerAPI.PlatformSecretsCreationFailedReason) } // create encryption bearer token for workloads per channel per banner err = r.createBearerToken(ctx, secretClient, banner, edgeencrypt.EncryptionJWTSecret, channel.ID.String(), channel.Name, fmt.Sprintf(edgeencrypt.EncryptionTokenSecretManager, channel.ID)) if err != nil { return recerr.New(err, bannerAPI.PlatformSecretsCreationFailedReason) } } return nil } // createKeyRing create key ring to store encryption keys func (r *EncryptionInfraReconciler) createKeyRing(ctx context.Context, bannerEdgeID string) error { return createKeyRing(ctx, r.KmsClient, r.kmsKey, bannerEdgeID) } // createKeyRing create key ring to store encryption keys func createKeyRing(ctx context.Context, kmsClient *kms.KeyManagementClient, kmsKey edgeencrypt.KmsKey, ring string) error { // check if key ring exists _, err := kmsClient.GetKeyRing(ctx, &kmspb.GetKeyRingRequest{Name: kmsKey.Ring(ring)}) if err != nil && status.Code(err) == codes.NotFound { _, err = kmsClient.CreateKeyRing(ctx, &kmspb.CreateKeyRingRequest{ Parent: kmsKey.RingParent(), KeyRingId: ring, }) } if err != nil { return fmt.Errorf("failed to create key ring: %w", err) } return nil } // createKMSSigningKey create encryption jwt signing key func (r *EncryptionInfraReconciler) createKMSSigningKey(ctx context.Context, secretClient secretManagerClient, banner *bannerAPI.Banner, channel, secretID string, labels map[string]string) error { return createKmsKey(ctx, r.KmsClient, secretClient, r.kmsKey, banner.Name, channel, labels, edgeencrypt.DefaultSigningCryptoKeyPurpose, edgeencrypt.DefaultSigningCryptoKeyAlgorithm, secretID, r.ResourceTimeout) } // createKMSEncryptionKey create encryption key func (r *EncryptionInfraReconciler) createKMSEncryptionKey(ctx context.Context, secretClient secretManagerClient, banner *bannerAPI.Banner, channel, secretID string, labels map[string]string) error { return createKmsKey(ctx, r.KmsClient, secretClient, r.kmsKey, banner.Name, channel, labels, edgeencrypt.DefaultCryptoKeyPurpose, edgeencrypt.DefaultCryptoKeyAlgorithm, secretID, r.ResourceTimeout) } func createKmsKey(ctx context.Context, kmsClient *kms.KeyManagementClient, secretClient secretManagerClient, kmsKey edgeencrypt.KmsKey, ring, key string, labels map[string]string, purpose kmspb.CryptoKey_CryptoKeyPurpose, algorithm kmspb.CryptoKeyVersion_CryptoKeyVersionAlgorithm, secretID string, timeout time.Duration) error { _, err := secretClient.GetSecret(ctx, secretID) if err == nil { // secret already exists, rotate will take care of the rest return nil } else if status.Code(err) != codes.NotFound { return fmt.Errorf("failed to get secret from secret manager: %w", err) } keyPath := kmsKey.KeyPath(ring, key, "1") pubKey, err := kmsClient.GetPublicKey(ctx, &kmspb.GetPublicKeyRequest{Name: keyPath}) if err != nil && status.Code(err) == codes.NotFound { _, err = kmsClient.CreateCryptoKey(ctx, &kmspb.CreateCryptoKeyRequest{ Parent: kmsKey.Ring(ring), CryptoKeyId: key, CryptoKey: &kmspb.CryptoKey{ Purpose: purpose, VersionTemplate: &kmspb.CryptoKeyVersionTemplate{Algorithm: algorithm}, Labels: labels, }, }) if err != nil { return fmt.Errorf("failed to create key ring: %w", err) } // get public key pubKey, err = getPublicKeyWithRetry(ctx, kmsClient, keyPath, timeout) } if err != nil { return fmt.Errorf("failed to get public key from kms: %w", err) } pk, err := edgeencrypt.NewPublicKey(pubKey.Pem, "1") if err != nil { return fmt.Errorf("failed to create public key: %w", err) } data, err := json.Marshal(pk) if err != nil { return fmt.Errorf("failed to marshal public key: %w", err) } // save public key to secret manager return saveToSecretManager(ctx, secretClient, secretID, data, 1, labels) } // createBearerToken create encryption bearer token func (r *EncryptionInfraReconciler) createBearerToken(ctx context.Context, secretClient secretManagerClient, banner *bannerAPI.Banner, keyName, channelID, channelName, secretID string) error { key := r.kmsKey.KeyPath(banner.Name, keyName, "1") return createBearerToken(ctx, secretClient, r.SigningMethod, key, channelID, channelName, secretID, edgeencrypt.Encryption, 1, map[string]string{"channel": channelName}, banner.Name) } func createBearerToken(ctx context.Context, secretClient secretManagerClient, sm jwt.SigningMethod, key string, channelID, channelName, secretID string, role edgeencrypt.Role, version int, labels map[string]string, banner ...string) error { _, err := secretClient.GetSecret(ctx, secretID) if err == nil { // secret already exists, rotate will take care of the rest return nil } else if status.Code(err) != codes.NotFound { return fmt.Errorf("failed to get secret from secret manager: %w", err) } token, err := edgeencrypt.CreateToken(sm, key, edgeencrypt.DefaultDuration, channelID, channelName, role, banner...) if err != nil { return fmt.Errorf("failed to create bearer token: %w", err) } t := &edgeencrypt.Token{BearerToken: token, Version: "1"} data, err := json.Marshal(t) if err != nil { return fmt.Errorf("failed to marshal bearer token to json: %w", err) } return saveToSecretManager(ctx, secretClient, secretID, data, version, labels) } func saveToSecretManager(ctx context.Context, secretClient secretManagerClient, secretID string, data []byte, _ int, labels map[string]string) error { // TODO fix version alias fmt.Sprintf("%d", version) err := secretClient.AddSecret(ctx, secretID, data, labels, false, nil, "") if err != nil { return fmt.Errorf("failed to add secret to secret manager: %w", err) } return nil } func (r *EncryptionInfraReconciler) CreateChannelKeyVersion(ctx context.Context, bannerEdgeID, channelID uuid.UUID, project, secretID string) error { ck := channels.ChannelKeyVersion{ ChannelID: channelID, BannerEdgeID: bannerEdgeID, Version: 1, SecretManagerLink: secretManagerLink(project, secretID), } return saveChannelKeyVersion(ctx, r.ChannelService, bannerEdgeID, channelID, ck) } func getPublicKeyWithRetry(ctx context.Context, kmsClient *kms.KeyManagementClient, key string, timeout time.Duration) (*kmspb.PublicKey, error) { var ( result *kmspb.PublicKey err error ) for { select { case <-time.After(timeout): if err != nil { return nil, err } return nil, fmt.Errorf("timeout waiting for key %s to be ready", key) default: result, err = kmsClient.GetPublicKey(ctx, &kmspb.GetPublicKeyRequest{Name: key}) if err == nil { return result, nil } time.Sleep(2 * time.Second) } } } func saveChannelKeyVersion(ctx context.Context, cs channels.Service, bannerEdgeID, channelID uuid.UUID, ckv channels.ChannelKeyVersion) error { _, err := cs.GetLatestChannelKeyVersion(ctx, bannerEdgeID, channelID) if err != nil { if !errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("failed to get latest channel key version: %w", err) } _, err = cs.CreateChannelKeyVersion(ctx, ckv) if err != nil { return fmt.Errorf("failed to create channel key version: %w", err) } } return nil } func secretManagerLink(project, secretID string) string { return fmt.Sprintf("projects/%s/secrets/%s", project, secretID) } func getBannerChannels(ctx context.Context, cs channels.Service, bannerEdgeID string) ([]channels.BannerChannel, error) { bannerID, err := uuid.Parse(bannerEdgeID) if err != nil { return nil, fmt.Errorf("failed to parse banner edge id: %w", err) } bcs, err := cs.GetBannerChannels(ctx, bannerID) if err != nil { return nil, fmt.Errorf("failed to get banner channels: %w", err) } return bcs, nil }