Source file src/edge-infra.dev/pkg/edge/controllers/bannerctl/bannerctl.go

Documentation: edge-infra.dev/pkg/edge/controllers/bannerctl

     1  package bannerctl
     2  
     3  import (
     4  	"context"
     5  	"encoding/base64"
     6  	"encoding/json"
     7  	"errors"
     8  	"fmt"
     9  	"os"
    10  	"reflect"
    11  	"strings"
    12  	"time"
    13  
    14  	registryAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/artifactregistry/v1beta1"
    15  	computeAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/compute/v1beta1"
    16  	containerAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/container/v1beta1"
    17  	iamAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/iam/v1beta1"
    18  	k8sAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/k8s/v1alpha1"
    19  	loggingAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/logging/v1beta1"
    20  	resourceAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/resourcemanager/v1beta1"
    21  	serviceAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/serviceusage/v1beta1"
    22  	storageAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/storage/v1beta1"
    23  	"github.com/fluxcd/pkg/ssa"
    24  	"github.com/go-logr/logr"
    25  	grpccodes "google.golang.org/grpc/codes"
    26  	grpcstatus "google.golang.org/grpc/status"
    27  	corev1 "k8s.io/api/core/v1"
    28  	kerrors "k8s.io/apimachinery/pkg/api/errors"
    29  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    30  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    31  	"k8s.io/apimachinery/pkg/runtime"
    32  	"k8s.io/apimachinery/pkg/runtime/schema"
    33  	"k8s.io/apimachinery/pkg/types"
    34  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    35  	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
    36  	"sigs.k8s.io/cli-utils/pkg/kstatus/polling"
    37  	ctrl "sigs.k8s.io/controller-runtime"
    38  	"sigs.k8s.io/controller-runtime/pkg/client"
    39  
    40  	kms "cloud.google.com/go/kms/apiv1"
    41  
    42  	bsltypes "edge-infra.dev/pkg/edge/api/bsl/types"
    43  	"edge-infra.dev/pkg/edge/api/services/channels"
    44  	"edge-infra.dev/pkg/edge/api/totp"
    45  	apitypes "edge-infra.dev/pkg/edge/api/types"
    46  	bannerAPI "edge-infra.dev/pkg/edge/apis/banner/v1alpha1"
    47  	edgeCluster "edge-infra.dev/pkg/edge/apis/cluster/v1alpha1"
    48  	edgeErrors "edge-infra.dev/pkg/edge/apis/errors"
    49  	sequelApi "edge-infra.dev/pkg/edge/apis/sequel/k8s/v1alpha2"
    50  	syncedobjectApi "edge-infra.dev/pkg/edge/apis/syncedobject/apis/v1alpha1"
    51  	"edge-infra.dev/pkg/edge/bsl"
    52  	"edge-infra.dev/pkg/edge/constants"
    53  	bannerconstants "edge-infra.dev/pkg/edge/constants/api/banner"
    54  	clusterConstants "edge-infra.dev/pkg/edge/constants/api/cluster"
    55  	"edge-infra.dev/pkg/edge/constants/api/fleet"
    56  	"edge-infra.dev/pkg/edge/controllers/dbmetrics"
    57  	"edge-infra.dev/pkg/edge/controllers/util/edgedb"
    58  	"edge-infra.dev/pkg/edge/edgeencrypt"
    59  	"edge-infra.dev/pkg/edge/flux/bootstrap"
    60  	"edge-infra.dev/pkg/edge/gcpinfra"
    61  	gcpconstants "edge-infra.dev/pkg/edge/gcpinfra/constants"
    62  	"edge-infra.dev/pkg/edge/k8objectsutils"
    63  	"edge-infra.dev/pkg/edge/registration"
    64  	"edge-infra.dev/pkg/edge/shipment/generator"
    65  	"edge-infra.dev/pkg/f8n/warehouse/cluster"
    66  	whv1 "edge-infra.dev/pkg/f8n/warehouse/k8s/apis/v1alpha1"
    67  	kccmeta "edge-infra.dev/pkg/k8s/konfigkonnector/apis/meta"
    68  	"edge-infra.dev/pkg/k8s/meta/status"
    69  	"edge-infra.dev/pkg/k8s/runtime/conditions"
    70  	"edge-infra.dev/pkg/k8s/runtime/controller"
    71  	"edge-infra.dev/pkg/k8s/runtime/controller/metrics"
    72  	"edge-infra.dev/pkg/k8s/runtime/controller/reconcile"
    73  	"edge-infra.dev/pkg/k8s/runtime/controller/reconcile/recerr"
    74  	"edge-infra.dev/pkg/k8s/runtime/inventory"
    75  	"edge-infra.dev/pkg/k8s/runtime/patch"
    76  	unstructuredutil "edge-infra.dev/pkg/k8s/unstructured"
    77  	"edge-infra.dev/pkg/lib/gcp/metricsscopes"
    78  	gcpProject "edge-infra.dev/pkg/lib/gcp/project"
    79  	"edge-infra.dev/pkg/lib/logging"
    80  	"edge-infra.dev/pkg/lib/uuid"
    81  )
    82  
    83  const (
    84  	projectName = "banner-project"
    85  	logSinkName = "siem"
    86  	dbInstance  = "-migrated"
    87  )
    88  
    89  var (
    90  	clusterMaxNodes    = 6
    91  	clusterMinNodes    = 1
    92  	clusterMachineType = "e2-standard-4"
    93  
    94  	// bannerConditions is the reconcile summarization configuration for how
    95  	// various conditions should be taken into account when the final condition is
    96  	// summarized
    97  	bannerConditions = reconcile.Conditions{
    98  		Target: status.ReadyCondition,
    99  		Owned: []string{
   100  			status.ReadyCondition,
   101  			status.ReconcilingCondition,
   102  			status.StalledCondition,
   103  		},
   104  		Summarize: []string{
   105  			status.StalledCondition,
   106  		},
   107  		NegativePolarity: []string{
   108  			status.ReconcilingCondition,
   109  			status.StalledCondition,
   110  		},
   111  	}
   112  )
   113  
   114  // ErrProjectNotReady occurs when the project isn't ready during reconciliation,
   115  // used to signal to the Reconciler that it needs to requeue
   116  var (
   117  	ErrProjectNotReady = errors.New("project is not ready")
   118  	ErrAPINotReady     = errors.New("gcp api is not ready")
   119  )
   120  
   121  //
   122  
   123  // +kubebuilder:rbac:groups=edge.ncr.com,resources=banners,verbs=*
   124  // +kubebuilder:rbac:groups=edge.ncr.com,resources=banners/status,verbs=*
   125  // +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;create
   126  // +kubebuilder:rbac:groups=edge.ncr.com,resources=clusters,verbs=create;get;list;update;patch;watch
   127  // +kubebuilder:rbac:groups=edge.ncr.com,resources=clusters/status,verbs=get;watch
   128  // +kubebuilder:rbac:groups=edge.ncr.com,resources=syncedobjects,verbs=*
   129  // +kubebuilder:rbac:groups=edge.ncr.com,resources=syncedobjects/status,verbs=get;watch
   130  // +kubebuilder:rbac:groups=backend.edge.ncr.com,resources=databaseusers,verbs=*
   131  // +kubebuilder:rbac:groups=backend.edge.ncr.com,resources=databaseusers/status,verbs=get;watch
   132  // +kubebuilder:rbac:groups="resourcemanager.cnrm.cloud.google.com",resources=projects,verbs=get;create;list;watch;patch
   133  // +kubebuilder:rbac:groups="resourcemanager.cnrm.cloud.google.com",resources=projects/status,verbs=get;watch
   134  // +kubebuilder:rbac:groups="container.cnrm.cloud.google.com",resources=containerclusters,verbs=get;create;list;watch;patch
   135  // +kubebuilder:rbac:groups="container.cnrm.cloud.google.com",resources=containerclusters/status,verbs=get;watch
   136  // +kubebuilder:rbac:groups="container.cnrm.cloud.google.com",resources=containernodepools,verbs=get;create;list;watch;patch
   137  // +kubebuilder:rbac:groups="container.cnrm.cloud.google.com",resources=containernodepools/status,verbs=get;watch
   138  // +kubebuilder:rbac:groups="iam.cnrm.cloud.google.com",resources=iamcustomroles,verbs=*
   139  // +kubebuilder:rbac:groups="iam.cnrm.cloud.google.com",resources=iamcustomroles/status,verbs=get;watch
   140  // +kubebuilder:rbac:groups="iam.cnrm.cloud.google.com",resources=iampolicymembers,verbs=*
   141  // +kubebuilder:rbac:groups="iam.cnrm.cloud.google.com",resources=iampolicymembers/status,verbs=get;watch
   142  // +kubebuilder:rbac:groups="iam.cnrm.cloud.google.com",resources=iamserviceaccounts,verbs=*
   143  // +kubebuilder:rbac:groups="iam.cnrm.cloud.google.com",resources=iamserviceaccounts/status,verbs=get;watch
   144  // +kubebuilder:rbac:groups="pubsub.cnrm.cloud.google.com",resources=pubsubtopics,verbs=*
   145  // +kubebuilder:rbac:groups="pubsub.cnrm.cloud.google.com",resources=pubsubtopics/status,verbs=get;watch
   146  // +kubebuilder:rbac:groups="pubsub.cnrm.cloud.google.com",resources=pubsubsubscriptions,verbs=*
   147  // +kubebuilder:rbac:groups="pubsub.cnrm.cloud.google.com",resources=pubsubsubscriptions/status,verbs=get;watch
   148  // +kubebuilder:rbac:groups="serviceusage.cnrm.cloud.google.com",resources=services,verbs=*
   149  // +kubebuilder:rbac:groups="serviceusage.cnrm.cloud.google.com",resources=services/status,verbs=get;watch
   150  // +kubebuilder:rbac:groups="compute.cnrm.cloud.google.com",resources=computenetworks,verbs=*
   151  // +kubebuilder:rbac:groups="compute.cnrm.cloud.google.com",resources=computenetworks/status,verbs=get;watch
   152  // +kubebuilder:rbac:groups="storage.cnrm.cloud.google.com",resources=storagebuckets,verbs=*
   153  // +kubebuilder:rbac:groups="storage.cnrm.cloud.google.com",resources=storagebuckets/status,verbs=get;watch
   154  // +kubebuilder:rbac:groups="compute.cnrm.cloud.google.com",resources=computeaddresses,verbs=*
   155  // +kubebuilder:rbac:groups="compute.cnrm.cloud.google.com",resources=computeaddresses/status,verbs=get;watch
   156  // +kubebuilder:rbac:groups="logging.cnrm.cloud.google.com",resources=logginglogexclusions,verbs=*
   157  // +kubebuilder:rbac:groups="logging.cnrm.cloud.google.com",resources=logginglogexclusions/status,verbs=get;watch
   158  // +kubebuilder:rbac:groups="logging.cnrm.cloud.google.com",resources=logginglogsinks,verbs=*
   159  // +kubebuilder:rbac:groups="logging.cnrm.cloud.google.com",resources=logginglogsinks/status,verbs=get;watch
   160  // +kubebuilder:rbac:groups="artifactregistry.cnrm.cloud.google.com",resources=artifactregistryrepositories,verbs=*
   161  // +kubebuilder:rbac:groups="artifactregistry.cnrm.cloud.google.com",resources=artifactregistryrepositories/status,verbs=get;watch
   162  // +kubebuilder:rbac:groups=warehouse.edge.ncr.com,resources=shipments,verbs=*
   163  
   164  type BannerReconciler struct {
   165  	client.Client
   166  	Log                      logr.Logger
   167  	Metrics                  metrics.Metrics
   168  	Conditions               reconcile.Conditions
   169  	BillingAccount           string
   170  	FolderID                 string
   171  	ForemanProjectID         string
   172  	PlatInfraProjectID       string
   173  	MetricsScopesClient      metricsScopesClient
   174  	SecretManager            secretManager
   175  	DefaultRequeue           time.Duration
   176  	Name                     string
   177  	ResourceManager          *ssa.ResourceManager
   178  	EdgeAPI                  string
   179  	TotpSecret               string
   180  	Domain                   string
   181  	DatasyncDNSName          string
   182  	DatasyncDNSZone          string
   183  	EdgeDB                   *edgedb.EdgeDB
   184  	DatabaseName             string
   185  	Recorder                 *dbmetrics.DBMetrics
   186  	BSLClient                *bsl.Client
   187  	BSLConfig                bsltypes.BSPConfig
   188  	GCPRegion                string
   189  	GCPForemanProjectNumber  string
   190  	GCPZone                  string
   191  	EdgeSecOptInCompliance   bool
   192  	EdgeSecMaxLeasePeriod    string
   193  	EdgeSecMaxValidityPeriod string
   194  }
   195  
   196  func Run(o ...controller.Option) error {
   197  	ctrl.SetLogger(logging.NewLogger().Logger)
   198  	log := ctrl.Log.WithName("setup")
   199  	cfg, _, err := NewConfig(os.Args)
   200  	if err != nil {
   201  		log.Error(err, "failed to parse startup configuration")
   202  		os.Exit(1)
   203  	}
   204  
   205  	mgr, err := create(cfg, o...)
   206  	if err != nil {
   207  		return err
   208  	}
   209  
   210  	log.Info("starting manager")
   211  	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
   212  		log.Error(err, "problem running manager")
   213  		return err
   214  	}
   215  
   216  	return nil
   217  }
   218  
   219  func create(cfg Config, o ...controller.Option) (ctrl.Manager, error) {
   220  	o = append(o, controller.WithMetricsAddress(cfg.MetricsAddr))
   221  	ctlCfg, opts := controller.ProcessOptions(o...)
   222  	opts.LeaderElectionID = "2c625a13.edge.ncr.com"
   223  	opts.Scheme = createScheme()
   224  
   225  	ctx := context.Background()
   226  	kmsClient, err := kms.NewKeyManagementClient(ctx)
   227  	if err != nil {
   228  		ctrl.Log.Error(err, "failed to create kms client")
   229  		return nil, err
   230  	}
   231  	sm := edgeencrypt.NewSigningMethodKMS(kmsClient)
   232  
   233  	err = CreateDecryptionInfra(ctx, kmsClient, sm, &gcpSecretManager{}, edgeencrypt.KmsKey{
   234  		ProjectID: cfg.ForemanProjectID,
   235  		Location:  cfg.GCPRegion,
   236  	}, cfg.ResourceTimeout)
   237  	if err != nil {
   238  		ctrl.Log.Error(err, "failed to create decryption infra")
   239  		return nil, err
   240  	}
   241  
   242  	dbm := dbmetrics.New("bannerctl")
   243  	mgr, err := ctrl.NewManager(ctlCfg, opts)
   244  
   245  	if err != nil {
   246  		ctrl.Log.Error(err, "failed to create manager")
   247  		return nil, err
   248  	}
   249  
   250  	bslClient := bsl.NewBSLClient(bsltypes.BSPConfig{
   251  		Endpoint:           cfg.BSLConfig.Endpoint,
   252  		Root:               cfg.BSLConfig.Root,
   253  		OrganizationPrefix: cfg.BSLConfig.OrganizationPrefix,
   254  	})
   255  	bslClient.SetDefaultAccessKey(cfg.BSLAccessKey.SharedKey, cfg.BSLAccessKey.SecretKey)
   256  	bslClient.SetTimeout(BSLDefaultTimeout)
   257  
   258  	m := metrics.New(mgr, "bannerctl", metrics.WithCollectors(dbm.Collectors()...))
   259  
   260  	if err = (&BannerReconciler{
   261  		Client:        mgr.GetClient(),
   262  		Log:           ctrl.Log.WithName("bannerctl"),
   263  		Metrics:       m,
   264  		Conditions:    bannerConditions,
   265  		SecretManager: &gcpSecretManager{},
   266  		MetricsScopesClient: &gcpMetricsScopesClient{
   267  			c: metricsscopes.New(cfg.ForemanProjectID),
   268  		},
   269  		ForemanProjectID:         cfg.ForemanProjectID,
   270  		PlatInfraProjectID:       cfg.PlatInfraProjectID,
   271  		DefaultRequeue:           10 * time.Second,
   272  		BillingAccount:           gcpconstants.DefaultBillingAccountID,
   273  		FolderID:                 cfg.ProjectBootstrapping.FolderID,
   274  		Name:                     "bannerctl",
   275  		EdgeAPI:                  cfg.EdgeAPI,
   276  		TotpSecret:               cfg.TotpSecret,
   277  		Domain:                   cfg.Domain,
   278  		DatasyncDNSName:          cfg.DatasyncDNSName,
   279  		DatasyncDNSZone:          cfg.DatasyncDNSZone,
   280  		EdgeDB:                   &edgedb.EdgeDB{DB: cfg.DB},
   281  		DatabaseName:             cfg.DatabaseName,
   282  		Recorder:                 dbm,
   283  		BSLClient:                bslClient,
   284  		BSLConfig:                cfg.BSLConfig,
   285  		GCPRegion:                cfg.GCPRegion,
   286  		GCPZone:                  cfg.GCPZone,
   287  		GCPForemanProjectNumber:  cfg.GCPForemanProjectNumber,
   288  		EdgeSecOptInCompliance:   cfg.EdgeSecOptInCompliance,
   289  		EdgeSecMaxLeasePeriod:    cfg.EdgeSecMaxLeasePeriod,
   290  		EdgeSecMaxValidityPeriod: cfg.EdgeSecMaxValidityPeriod,
   291  	}).SetupWithManager(mgr); err != nil {
   292  		ctrl.Log.Error(err, "failed to create controller and set up with manager")
   293  		return nil, err
   294  	}
   295  
   296  	cs := channels.NewChannelService(cfg.DB, cfg.ForemanProjectID, nil)
   297  
   298  	if err = (&EncryptionInfraReconciler{
   299  		Client:           mgr.GetClient(),
   300  		Log:              ctrl.Log.WithName("encryptioninfra"),
   301  		Conditions:       bannerConditions,
   302  		SecretManager:    &gcpSecretManager{},
   303  		KmsClient:        kmsClient,
   304  		ForemanProjectID: cfg.ForemanProjectID,
   305  		GCPRegion:        cfg.GCPRegion,
   306  		IntervalTime:     cfg.IntervalTime,
   307  		RequeueTime:      cfg.RequeueTime,
   308  		ResourceTimeout:  cfg.ResourceTimeout,
   309  		ChannelService:   cs,
   310  		SigningMethod:    sm,
   311  	}).SetupWithManager(mgr); err != nil {
   312  		ctrl.Log.Error(err, "failed to create EncryptionInfra controller and set up with manager")
   313  		return nil, err
   314  	}
   315  	if err = (&EncryptionKeyManagementReconciler{
   316  		Client:           mgr.GetClient(),
   317  		Log:              ctrl.Log.WithName("encryptionkeymanagement"),
   318  		Conditions:       bannerConditions,
   319  		SecretManager:    &gcpSecretManager{},
   320  		KmsClient:        kmsClient,
   321  		ForemanProjectID: cfg.ForemanProjectID,
   322  		GCPRegion:        cfg.GCPRegion,
   323  		IntervalTime:     cfg.IntervalTime,
   324  		RequeueTime:      cfg.RequeueTime,
   325  		ResourceTimeout:  cfg.ResourceTimeout,
   326  		ChannelService:   cs,
   327  	}).SetupWithManager(mgr); err != nil {
   328  		ctrl.Log.Error(err, "failed to create controller and set up with manager")
   329  		return nil, err
   330  	}
   331  	return mgr, nil
   332  }
   333  
   334  func (r *BannerReconciler) SetupWithManager(mgr ctrl.Manager) error {
   335  	return ctrl.NewControllerManagedBy(mgr).
   336  		For(&bannerAPI.Banner{}).
   337  		// leaving this here as we'll likely go back to something similar in the future
   338  		// For(&bannerAPI.Banner{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
   339  		Owns(&resourceAPI.Project{}).
   340  		Owns(&edgeCluster.Cluster{}).
   341  		Owns(&containerAPI.ContainerNodePool{}).
   342  		Owns(&iamAPI.IAMCustomRole{}).
   343  		Owns(&iamAPI.IAMPolicyMember{}).
   344  		Owns(&serviceAPI.Service{}).
   345  		Owns(&storageAPI.StorageBucket{}).
   346  		Owns(&syncedobjectApi.SyncedObject{}).
   347  		Owns(&loggingAPI.LoggingLogExclusion{}).
   348  		Owns(&registryAPI.ArtifactRegistryRepository{}).
   349  		Owns(&sequelApi.DatabaseUser{}).
   350  		Complete(r)
   351  }
   352  
   353  func (r *BannerReconciler) PatchOpts() []patch.Option {
   354  	return []patch.Option{
   355  		patch.WithOwnedConditions{Conditions: r.Conditions.Owned},
   356  		patch.WithFieldOwner(r.Name),
   357  	}
   358  }
   359  
   360  func createScheme() *runtime.Scheme {
   361  	scheme := runtime.NewScheme()
   362  	utilruntime.Must(bannerAPI.AddToScheme(scheme))
   363  	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
   364  	utilruntime.Must(containerAPI.AddToScheme(scheme))
   365  	utilruntime.Must(computeAPI.AddToScheme(scheme))
   366  	utilruntime.Must(containerAPI.AddToScheme(scheme))
   367  	utilruntime.Must(edgeCluster.AddToScheme(scheme))
   368  	utilruntime.Must(iamAPI.AddToScheme(scheme))
   369  	utilruntime.Must(loggingAPI.AddToScheme(scheme))
   370  	utilruntime.Must(resourceAPI.AddToScheme(scheme))
   371  	utilruntime.Must(serviceAPI.AddToScheme(scheme))
   372  	utilruntime.Must(storageAPI.AddToScheme(scheme))
   373  	utilruntime.Must(syncedobjectApi.AddToScheme(scheme))
   374  	utilruntime.Must(loggingAPI.AddToScheme(scheme))
   375  	utilruntime.Must(registryAPI.AddToScheme(scheme))
   376  	utilruntime.Must(sequelApi.AddToScheme(scheme))
   377  	return scheme
   378  }
   379  
   380  func (r *BannerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, recErr error) {
   381  	var (
   382  		reconcileStart = time.Now()
   383  		log            = ctrl.LoggerFrom(ctx)
   384  		result         = reconcile.ResultEmpty
   385  		banner         = &bannerAPI.Banner{}
   386  	)
   387  	r.setResourceManager()
   388  
   389  	if err := r.Get(ctx, req.NamespacedName, banner); err != nil {
   390  		return ctrl.Result{}, client.IgnoreNotFound(err)
   391  	}
   392  	r.Metrics.RecordReconciling(ctx, banner)
   393  
   394  	patcher := patch.NewSerialPatcher(banner, r.Client)
   395  
   396  	defer func() {
   397  		summarizer := reconcile.NewSummarizer(patcher)
   398  		res, recErr = summarizer.SummarizeAndPatch(ctx, banner, []reconcile.SummarizeOption{
   399  			reconcile.WithConditions(r.Conditions),
   400  			reconcile.WithResult(result),
   401  			reconcile.WithError(recErr),
   402  			reconcile.WithIgnoreNotFound(),
   403  			reconcile.WithProcessors(
   404  				reconcile.RecordReconcileReq,
   405  				reconcile.RecordResult,
   406  			),
   407  			reconcile.WithFieldOwner(r.Name),
   408  		}...)
   409  
   410  		r.Metrics.RecordDuration(ctx, banner, reconcileStart)
   411  		r.Metrics.RecordReadiness(ctx, banner)
   412  		r.EdgeDB.RecordInfraStatus(ctx, banner, *r.Recorder)
   413  	}()
   414  
   415  	// Banner fetched successfully, so decorate logger with basic banner info
   416  	// and create a new context to store updated logger
   417  	log.WithValues("banner", banner.Spec.DisplayName, "bannerEdgeID", banner.Name)
   418  	ctx = logr.NewContext(ctx, log)
   419  	log.Info("reconciling started for banner")
   420  
   421  	recErr = r.reconcile(ctx, patcher, banner)
   422  	if recErr == nil {
   423  		result = reconcile.ResultSuccess
   424  	}
   425  	return
   426  }
   427  
   428  // reconcile handles the actual reconciliation of the banner object
   429  func (r *BannerReconciler) reconcile(ctx context.Context, patcher *patch.SerialPatcher, b *bannerAPI.Banner) recerr.Error {
   430  	log := ctrl.LoggerFrom(ctx)
   431  
   432  	// Validate Banner spec
   433  	if err := b.IsValid(); err != nil {
   434  		// TODO: validatingadmissionwebhook
   435  		log.Error(err, "invalid banner object")
   436  		return recerr.NewStalled(fmt.Errorf("invalid spec: %w", err), bannerAPI.InvalidSpecReason)
   437  	}
   438  
   439  	if err := reconcile.Progressing(ctx, b, patcher, r.PatchOpts()...); err != nil {
   440  		return recerr.New(err, bannerAPI.ReconcileFailedReason)
   441  	}
   442  
   443  	recErr := r.reconcileNamespace(ctx, b)
   444  	if recErr != nil {
   445  		return recErr
   446  	}
   447  
   448  	// Create project so we can use the number to create dependent resources
   449  	recErr = r.reconcileProject(ctx, b)
   450  	if recErr != nil {
   451  		// Reflect error for Project on Ready condition
   452  		recErr.ToCondition(b, status.ReadyCondition)
   453  		return recErr
   454  	}
   455  
   456  	// Project created successfully, so decorate logger with the projectID
   457  	// and create a new context to store updated logger
   458  	ctx = logr.NewContext(ctx, log.WithValues("projectID", b.Spec.GCP.ProjectID))
   459  
   460  	recErr = r.reconcileProjectInfra(ctx, b)
   461  	if recErr != nil {
   462  		// Reflect error for Project infra on Ready condition
   463  		recErr.ToCondition(b, status.ReadyCondition)
   464  		return recErr
   465  	}
   466  
   467  	recErr = r.reconcilePlatformSecrets(ctx, b)
   468  	if recErr != nil {
   469  		// Reflect error for Project infra on Ready condition
   470  		recErr.ToCondition(b, status.ReadyCondition)
   471  		return recErr
   472  	}
   473  
   474  	recErr = r.reconcileAutomatedEdgeLabels(ctx, b)
   475  	if recErr != nil {
   476  		// Reflect error for Project infra on Ready condition
   477  		recErr.ToCondition(b, status.ReadyCondition)
   478  		return recErr
   479  	}
   480  
   481  	recErr = r.reconcileCerts(ctx, b)
   482  	if recErr != nil {
   483  		log.Error(recErr, "failed to reconcile certificates")
   484  		// Reflect error for Project infra on Ready condition
   485  		recErr.ToCondition(b, status.ReadyCondition)
   486  		return recErr
   487  	}
   488  
   489  	err := r.addMetricsScopes(ctx, b)
   490  	if err != nil {
   491  		recErr = recerr.New(err, bannerAPI.ProjectSetupFailedReason)
   492  		// Reflect error for Project infra on Ready condition
   493  		recErr.ToCondition(b, status.ReadyCondition)
   494  		return recErr
   495  	}
   496  	log.Info("banner reconciled successfully")
   497  
   498  	conditions.MarkTrue(b, status.ReadyCondition, bannerAPI.ProvisionSucceededReason, "banner reconciled successfully")
   499  	return nil
   500  }
   501  
   502  func (r *BannerReconciler) reconcileNamespace(ctx context.Context, b *bannerAPI.Banner) recerr.Error {
   503  	log := ctrl.LoggerFrom(ctx).WithName("namespace")
   504  
   505  	namespace := &corev1.Namespace{
   506  		ObjectMeta: metav1.ObjectMeta{
   507  			Name:            b.Name,
   508  			OwnerReferences: r.ownerRef(b),
   509  			Annotations: map[string]string{
   510  				kccmeta.DeletionPolicyAnnotation: kccmeta.DeletionPolicyAbandon,
   511  			},
   512  		},
   513  	}
   514  	kccmeta.SetProjectAnnotation(&namespace.ObjectMeta, b.Spec.GCP.ProjectID)
   515  	err := client.IgnoreAlreadyExists(r.Create(ctx, namespace, r.createOpts()))
   516  	if err != nil {
   517  		log.Error(err, "failed to create namespace")
   518  		return recerr.New(err, bannerAPI.NamespaceCreationFailedReason)
   519  	}
   520  	log.Info("namespace created", "namespace", namespace.Name, "ownerReferences", namespace.OwnerReferences)
   521  
   522  	return nil
   523  }
   524  
   525  func (r *BannerReconciler) createSiemSink(b *bannerAPI.Banner) *loggingAPI.LoggingLogSink {
   526  	description := "Route security relevant logs to a siem storage bucket"
   527  	sinkFilter := "jsonPayload.enable_siem='true'"
   528  	storageBucketName := fmt.Sprintf("%s-%s", r.ForemanProjectID, logSinkName)
   529  	// bucketRef will be: storage.googleapis.com/ret-edge-dev1-foreman-siem
   530  	// https://cloud.google.com/config-connector/docs/reference/resource-docs/logging/logginglogsink#spec
   531  	bucketRef := fmt.Sprintf("%s/%s", "storage.googleapis.com", storageBucketName)
   532  	uniqueWriter := true
   533  
   534  	return &loggingAPI.LoggingLogSink{
   535  		ObjectMeta: metav1.ObjectMeta{
   536  			Name:            logSinkName,
   537  			Namespace:       b.Name,
   538  			OwnerReferences: r.ownerRef(b),
   539  		},
   540  		TypeMeta: gvkToTypeMeta(loggingAPI.LoggingLogSinkGVK),
   541  		Spec: loggingAPI.LoggingLogSinkSpec{
   542  			ProjectRef: &k8sAPI.ResourceRef{
   543  				External: b.Spec.GCP.ProjectID,
   544  			},
   545  			Destination: loggingAPI.LogsinkDestination{
   546  				StorageBucketRef: &k8sAPI.ResourceRef{
   547  					External: bucketRef,
   548  				},
   549  			},
   550  			Description:          &description,
   551  			Filter:               &sinkFilter,
   552  			UniqueWriterIdentity: &uniqueWriter,
   553  		},
   554  	}
   555  }
   556  
   557  // createInfraSAResources generates iam service account resources
   558  func (r *BannerReconciler) createInfraSAResources(b *bannerAPI.Banner) []client.Object {
   559  	hash := uuid.FromUUID(b.Status.ClusterInfraClusterEdgeID).Hash()
   560  	kccResourceName := fmt.Sprintf("kcc-%s", hash)
   561  	clusterctlSAName := fmt.Sprintf("cctl-%s", hash)
   562  	soctlSAName := fmt.Sprintf("soctl-%s", hash)
   563  	projectNumber := b.Status.ProjectNumber
   564  
   565  	var objs []client.Object
   566  	objs = append(objs, r.createClusterInfraKCCResources(b, kccResourceName, projectNumber)...)
   567  	objs = append(objs, r.createClusterControllerSAResources(b, clusterctlSAName)...)
   568  	objs = append(objs, r.createSyncedObjectCtlSAResources(b, soctlSAName)...)
   569  	return objs
   570  }
   571  
   572  // reconcileProjectInfra creates the GCP Project and required KCC resources
   573  func (r *BannerReconciler) reconcileProjectInfra(ctx context.Context, b *bannerAPI.Banner) recerr.Error {
   574  	var (
   575  		err             error
   576  		mgr             = r.ResourceManager
   577  		log             = ctrl.LoggerFrom(ctx).WithName("project-infra")
   578  		oldBannerStatus = b.Status.DeepCopy()
   579  	)
   580  
   581  	err = r.createClusterInfraCluster(ctx, b)
   582  	if err != nil {
   583  		log.Error(err, fmt.Sprintf("failed to call registration api for banner %s cluster-infra cluster", b.Spec.DisplayName))
   584  		return recerr.New(err, bannerAPI.ClusterInfraCreationFailedReason)
   585  	}
   586  
   587  	// otherwise, the project is ready and we are good to go
   588  	objs := []client.Object{
   589  		r.createStorageBucket(b),
   590  		r.createSiemSink(b),
   591  		r.createClusterctlDatabaseUser(b),
   592  		r.createEdgeIssuerDatabaseUser(b),
   593  		r.createAuthserverDatabaseUser(b),
   594  	}
   595  
   596  	roles := r.createCustomStorageRoles(b)
   597  	for _, role := range roles {
   598  		objs = append(objs, role)
   599  	}
   600  	svcs := r.createAPIEnablements(b)
   601  	for _, svc := range svcs {
   602  		objs = append(objs, svc)
   603  	}
   604  
   605  	// resources for cluster-infra in tenant project
   606  	objs = append(objs, r.createInfraSAResources(b)...)
   607  
   608  	// add banner-wide namespace to create the top-level 'chariot' folder
   609  	nsObj, err := createBannerWideNamespace(b)
   610  	if err != nil {
   611  		log.Error(err, fmt.Sprintf("failed to create banner-wide namespace for %s", b.Spec.DisplayName))
   612  		return recerr.New(err, bannerAPI.ApplyFailedReason)
   613  	}
   614  	objs = append(objs, nsObj)
   615  
   616  	// create remote access synced objects
   617  	objs = append(objs, r.createRemoteAccessComputeAddress(ctx, b))
   618  	objs = append(objs, r.genForemanSO(r.createForemanProxyMapping(b), b))
   619  
   620  	// optional enablements
   621  	for _, e := range b.Spec.Enablements {
   622  		if e == CouchDBEnablement {
   623  			err = r.createCouchServerCluster(ctx, b)
   624  			if err != nil {
   625  				log.Error(err, fmt.Sprintf("failed to call registration api for banner %s couch cluster", b.Spec.DisplayName))
   626  				return recerr.New(err, bannerAPI.CouchClusterCreationFailedReason)
   627  			}
   628  			err = r.bslFullSync(ctx, b)
   629  			if err != nil {
   630  				log.Error(err, fmt.Sprintf("fail to sync bsl data to couchdb banner %s", b.Spec.DisplayName))
   631  				return recerr.New(err, bannerAPI.CouchBSLDataSyncFailedReason)
   632  			}
   633  			shipments, err := r.generateShipments(b)
   634  			if err != nil {
   635  				return recerr.NewStalled(err, bannerAPI.InvalidShipmentSpecReason)
   636  			}
   637  			objs = append(objs, shipments...)
   638  
   639  			// BSL EU ID is needed to move this to couchdb-masters cluster pallet
   640  			objs = append(objs, r.createCouchCushionConfigMapSO(b))
   641  		}
   642  	}
   643  
   644  	// warehouse infrastructure
   645  	objs = append(objs, r.createGARRepo(b))
   646  
   647  	var unstructuredObjs []*unstructured.Unstructured
   648  	for _, obj := range objs {
   649  		uobj, err := unstructuredutil.ToUnstructured(obj)
   650  		if err != nil {
   651  			goodErr := fmt.Errorf("failed to convert %s/%s/%s to unstructured: %w", obj.GetObjectKind(), obj.GetNamespace(), obj.GetName(), err)
   652  			return recerr.New(goodErr, bannerAPI.ApplyFailedReason)
   653  		}
   654  		unstructuredObjs = append(unstructuredObjs, uobj)
   655  	}
   656  
   657  	changeSet, err := mgr.ApplyAll(ctx, unstructuredObjs, ssa.ApplyOptions{Force: true})
   658  	if err != nil {
   659  		return recerr.New(err, bannerAPI.ApplyFailedReason)
   660  	}
   661  	log.Info("project infrastructure applied", "changeset", changeSet.ToMap())
   662  	b.Status.Inventory = inventory.New(inventory.FromChangeSet(changeSet))
   663  
   664  	if oldBannerStatus.Inventory != nil {
   665  		diff, err := inventory.Diff(oldBannerStatus.Inventory, b.GetInventory())
   666  		if err != nil {
   667  			return recerr.New(err, bannerAPI.PruneFailedReason)
   668  		}
   669  		log.Info("inventory", "diff", diff)
   670  		if len(diff) > 0 {
   671  			changeSet, err := r.ResourceManager.DeleteAll(ctx, diff, ssa.DefaultDeleteOptions())
   672  			if err != nil {
   673  				return recerr.New(err, bannerAPI.PruneFailedReason)
   674  			}
   675  			log.Info("pruned objects", "changeset", changeSet.ToMap())
   676  		}
   677  	}
   678  	return nil
   679  }
   680  
   681  func (r *BannerReconciler) reconcileProject(ctx context.Context, b *bannerAPI.Banner) recerr.Error {
   682  	var (
   683  		err     error
   684  		project *resourceAPI.Project
   685  		log     = ctrl.LoggerFrom(ctx).WithName("project")
   686  	)
   687  
   688  	// TODO: validate project shape in validationadmissionwebhook
   689  	project, err = r.createProject(b)
   690  	if err != nil {
   691  		log.Error(err, "failed to declare project resource")
   692  		return recerr.New(err, bannerAPI.ProjectSetupFailedReason)
   693  	}
   694  
   695  	if err := client.IgnoreAlreadyExists(r.Create(ctx, project)); err != nil {
   696  		log.Error(err, "failed to create project")
   697  		return recerr.New(err, bannerAPI.ProjectSetupFailedReason)
   698  	}
   699  
   700  	// Project resource now exists, even if its not ready
   701  	b.Status.ProjectRef = fmt.Sprintf("%s/%s", project.Namespace, project.Name)
   702  
   703  	if err := r.Get(ctx, types.NamespacedName{Name: project.Name, Namespace: project.Namespace}, project); err != nil {
   704  		log.Error(err, "failed to get project")
   705  		return recerr.New(err, bannerAPI.ProjectSetupFailedReason)
   706  	}
   707  
   708  	if ready, reason := kccmeta.IsReady(project.Status.Conditions); !ready || project.Status.Number == nil {
   709  		log.Info("project is not ready", "reason", reason)
   710  		return recerr.NewWait(fmt.Errorf("%w: %s", ErrProjectNotReady, reason), bannerAPI.ProjectNotReadyReason, r.DefaultRequeue)
   711  	}
   712  
   713  	b.Status.ProjectNumber = *project.Status.Number
   714  	log.Info("project is ready", "project", project.ObjectMeta)
   715  
   716  	return nil
   717  }
   718  
   719  func (r *BannerReconciler) createProject(b *bannerAPI.Banner) (*resourceAPI.Project, error) {
   720  	if b.Spec.GCP.ProjectID == "" {
   721  		b.Spec.GCP.ProjectID = fmt.Sprintf("%s-%s", gcpinfra.ProjectIDPrefix, gcpProject.RandAN(29-(len(gcpinfra.ProjectIDPrefix))))
   722  	}
   723  
   724  	// TODO: validate project shape in validationadmissionwebhook
   725  	if err := gcpProject.IsValidProjectID(b.Spec.GCP.ProjectID); err != nil {
   726  		return &resourceAPI.Project{}, err
   727  	}
   728  
   729  	return &resourceAPI.Project{
   730  		ObjectMeta: metav1.ObjectMeta{
   731  			Name:      projectName,
   732  			Namespace: b.Name,
   733  			Annotations: map[string]string{
   734  				constants.Banner:          b.Spec.DisplayName,
   735  				constants.Organization:    b.Spec.BSL.Organization.Name,
   736  				kccmeta.ProjectAnnotation: b.Spec.GCP.ProjectID,
   737  			},
   738  			OwnerReferences: r.ownerRef(b),
   739  		},
   740  		TypeMeta: gvkToTypeMeta(resourceAPI.ProjectGVK),
   741  		Spec: resourceAPI.ProjectSpec{
   742  			BillingAccountRef: kccmeta.BillingAccountRef(r.BillingAccount),
   743  			Name:              b.Spec.DisplayName,
   744  			ResourceID:        &b.Spec.GCP.ProjectID,
   745  			FolderRef:         kccmeta.FolderRef(r.FolderID),
   746  		},
   747  	}, nil
   748  }
   749  
   750  func (r *BannerReconciler) createCustomStorageRoles(b *bannerAPI.Banner) []*iamAPI.IAMCustomRole {
   751  	iamRoleDescriptionGet := "IAM role to get bucket and object metadata needed for flux source controller"
   752  	iamRoleDescriptionList := "IAM role to list bucket objects needed for flux source controller"
   753  	fluxRoleGet := &iamAPI.IAMCustomRole{
   754  		ObjectMeta: metav1.ObjectMeta{
   755  			Name:      bootstrap.FluxRoleGet,
   756  			Namespace: b.Name,
   757  			Annotations: map[string]string{
   758  				kccmeta.DeletionPolicyAnnotation: kccmeta.DeletionPolicyAbandon,
   759  			},
   760  			OwnerReferences: r.ownerRef(b),
   761  		},
   762  		Spec: iamAPI.IAMCustomRoleSpec{
   763  			Description: &iamRoleDescriptionGet,
   764  			Permissions: []string{"storage.objects.get"},
   765  			Title:       bootstrap.FluxRoleGet,
   766  		},
   767  		TypeMeta: gvkToTypeMeta(iamAPI.IAMCustomRoleGVK),
   768  	}
   769  
   770  	fluxRoleList := &iamAPI.IAMCustomRole{
   771  		ObjectMeta: metav1.ObjectMeta{
   772  			Name:      bootstrap.FluxRoleList,
   773  			Namespace: b.Name,
   774  			Annotations: map[string]string{
   775  				kccmeta.DeletionPolicyAnnotation: kccmeta.DeletionPolicyAbandon,
   776  			},
   777  			OwnerReferences: r.ownerRef(b),
   778  		},
   779  		Spec: iamAPI.IAMCustomRoleSpec{
   780  			Description: &iamRoleDescriptionList,
   781  			Permissions: []string{"storage.objects.list", "storage.buckets.get"},
   782  			Title:       bootstrap.FluxRoleList,
   783  		},
   784  		TypeMeta: gvkToTypeMeta(iamAPI.IAMCustomRoleGVK),
   785  	}
   786  
   787  	return []*iamAPI.IAMCustomRole{
   788  		fluxRoleGet,
   789  		fluxRoleList,
   790  	}
   791  }
   792  
   793  func (r *BannerReconciler) createAPIEnablements(b *bannerAPI.Banner) []*serviceAPI.Service {
   794  	// svcMeta will be applied to all Service CRDs we create
   795  	svcMeta := metav1.ObjectMeta{
   796  		Annotations: map[string]string{
   797  			kccmeta.DisableDepSvcAnnotation:       "false",
   798  			kccmeta.DisableSvcOnDestroyAnnotation: "false",
   799  			kccmeta.DeletionPolicyAnnotation:      kccmeta.DeletionPolicyAbandon,
   800  		},
   801  		Namespace:       b.Name,
   802  		OwnerReferences: r.ownerRef(b),
   803  	}
   804  	var objs []*serviceAPI.Service
   805  	// create a service object with the common metadata for each API the control
   806  	// plane needs enabled
   807  	for _, api := range gcpinfra.TenantAPIs {
   808  		svcMeta.Name = api
   809  		objs = append(objs, &serviceAPI.Service{
   810  			ObjectMeta: svcMeta,
   811  			TypeMeta:   gvkToTypeMeta(serviceAPI.ServiceGVK),
   812  		})
   813  	}
   814  
   815  	return objs
   816  }
   817  
   818  func (r *BannerReconciler) checkAPI(ctx context.Context, ns string, api string) (bool, error) {
   819  	log := ctrl.LoggerFrom(ctx).WithName("check-API")
   820  
   821  	service := &serviceAPI.Service{}
   822  	err := r.Client.Get(ctx, types.NamespacedName{Name: api, Namespace: ns}, service)
   823  	if err != nil && !kerrors.IsNotFound(err) {
   824  		log.Error(err, "Error checking api enablement", "service name", api)
   825  		return false, err
   826  	}
   827  	ready, reason := kccmeta.IsReady(service.Status.Conditions)
   828  	if !ready {
   829  		log.Info("waiting for service api to become Ready", "reason", reason, "api", api)
   830  	}
   831  	return ready, nil
   832  }
   833  
   834  func (r *BannerReconciler) copyPlatformSecrets(ctx context.Context, projectID string) error {
   835  	log := ctrl.LoggerFrom(ctx).WithName("copy-platform-secrets").WithValues("projectID", projectID)
   836  
   837  	log.Info("copying platform secrets to new banner project", "source", r.ForemanProjectID, "destination", projectID)
   838  	reader, err := r.SecretManager.NewWithOptions(ctx, r.ForemanProjectID)
   839  	if err != nil {
   840  		return fmt.Errorf("error creating secretmanager reader client, err: %v", err)
   841  	}
   842  	writer, err := r.SecretManager.NewWithOptions(ctx, projectID)
   843  	if err != nil {
   844  		return fmt.Errorf("error creating secretmanager writer client, err: %v", err)
   845  	}
   846  	for _, secretID := range constants.PlatformSecretIDs {
   847  		secretVal, err := reader.GetLatestSecretValue(ctx, secretID)
   848  		if err != nil {
   849  			return fmt.Errorf("error reading latest secret value, secretID: %v, err: %v", secretID, err)
   850  		}
   851  		s, err := reader.GetSecret(ctx, secretID)
   852  		if err != nil {
   853  			return fmt.Errorf("error getting secret, secretID: %v, err: %v", secretID, err)
   854  		}
   855  
   856  		err = writer.AddSecret(ctx, secretID, secretVal, s.Labels, true, nil, "")
   857  		if err != nil {
   858  			return fmt.Errorf("error adding secret, secretID: %v, labels: %v, err: %v", secretVal, s.Labels, err)
   859  		}
   860  	}
   861  	log.Info("copied platform secrets to new banner project")
   862  	return nil
   863  }
   864  
   865  func (r *BannerReconciler) createStorageBucket(b *bannerAPI.Banner) *storageAPI.StorageBucket {
   866  	return &storageAPI.StorageBucket{
   867  		ObjectMeta: metav1.ObjectMeta{
   868  			Name:      b.Spec.GCP.ProjectID,
   869  			Namespace: b.Name,
   870  			Annotations: map[string]string{
   871  				kccmeta.DeletionPolicyAnnotation: kccmeta.DeletionPolicyAbandon,
   872  			},
   873  			OwnerReferences: r.ownerRef(b),
   874  		},
   875  		TypeMeta: gvkToTypeMeta(storageAPI.StorageBucketGVK),
   876  		Spec: storageAPI.StorageBucketSpec{
   877  			Versioning: &storageAPI.BucketVersioning{Enabled: true},
   878  		},
   879  	}
   880  }
   881  
   882  func (r *BannerReconciler) addMetricsScopes(ctx context.Context, b *bannerAPI.Banner) error {
   883  	log := ctrl.LoggerFrom(ctx).WithName("metrics-scopes")
   884  
   885  	resp, err := r.MetricsScopesClient.AddMonitoredProject(ctx, b.Spec.GCP.ProjectID)
   886  	if err != nil && grpcstatus.Code(err) != grpccodes.AlreadyExists {
   887  		log.Error(err, "failed to register monitored project", "host-project", r.ForemanProjectID, "response", resp)
   888  		return err
   889  	}
   890  	return nil
   891  }
   892  
   893  func (r *BannerReconciler) createClusterInfraCluster(ctx context.Context, b *bannerAPI.Banner) error {
   894  	// Check if cluster resource already exists
   895  	if b.Status.ClusterInfraClusterEdgeID != "" {
   896  		return nil
   897  	}
   898  	// If not, call registration api for cluster
   899  	totpToken, err := totp.GenerateTotp(r.TotpSecret)
   900  	if err != nil {
   901  		return err
   902  	}
   903  	GCPLocation := fmt.Sprintf("%s-%s", r.GCPRegion, r.GCPZone)
   904  	reg, err := registration.NewBuilder().
   905  		Banner(b.Spec.DisplayName).
   906  		Store(bannerconstants.CreateBannerClusterInfraName(b.Spec.DisplayName)).
   907  		ClusterType(clusterConstants.GKE).
   908  		BSLOrganization(b.Spec.BSL.Organization.Name).
   909  		APIEndpoint(r.EdgeAPI). //todo
   910  		TotpToken(totpToken.Code).
   911  		CreateBSLSite(false).
   912  		Fleet(fleet.Cluster).
   913  		MachineType(clusterMachineType).
   914  		MinNodes(clusterMinNodes).
   915  		MaxNodes(clusterMaxNodes).
   916  		Autoscale(true).
   917  		Location(GCPLocation).
   918  		FleetVersion(apitypes.DefaultVersionTag).
   919  		Build()
   920  	if err != nil {
   921  		return err
   922  	}
   923  	reg.Client = r.Client
   924  	registrationResponse, err := reg.RegisterCluster(ctx)
   925  	if err != nil && !strings.Contains(err.Error(), edgeErrors.ErrClusterAlreadyExists) {
   926  		return err
   927  	}
   928  	if registrationResponse != nil {
   929  		b.Status.ClusterInfraClusterEdgeID = registrationResponse.ClusterEdgeID
   930  		// New cluster infra clusters the project id will be the banner project id
   931  		b.Status.ClusterInfraClusterProjectID = b.Spec.GCP.ProjectID
   932  	}
   933  	return nil
   934  }
   935  
   936  func (r *BannerReconciler) createAuthserverDatabaseUser(b *bannerAPI.Banner) *sequelApi.DatabaseUser {
   937  	hash := uuid.FromUUID(b.Status.ClusterInfraClusterEdgeID).Hash()
   938  	authserverName := fmt.Sprintf("authserver-%s", hash)
   939  	iamUsername := fmt.Sprintf("authserver@%s.iam", b.Spec.GCP.ProjectID)
   940  	grant := sequelApi.Grant{
   941  		Schema: "public",
   942  		TableGrant: []sequelApi.TableGrant{
   943  			{
   944  				Table: "tenants",
   945  				Permissions: []sequelApi.Permissions{
   946  					{
   947  						Permission: "SELECT",
   948  					},
   949  				},
   950  			},
   951  			{
   952  				Table: "banners",
   953  				Permissions: []sequelApi.Permissions{
   954  					{
   955  						Permission: "SELECT",
   956  					},
   957  				},
   958  			},
   959  			{
   960  				Table: "banner_configs",
   961  				Permissions: []sequelApi.Permissions{
   962  					{
   963  						Permission: "SELECT",
   964  					},
   965  				},
   966  			},
   967  			{
   968  				Table: "clusters",
   969  				Permissions: []sequelApi.Permissions{
   970  					{
   971  						Permission: "SELECT",
   972  					},
   973  				},
   974  			},
   975  			{
   976  				Table: "cluster_config",
   977  				Permissions: []sequelApi.Permissions{
   978  					{
   979  						Permission: "SELECT",
   980  					},
   981  				},
   982  			},
   983  			{
   984  				Table: "http_sessions",
   985  				Permissions: []sequelApi.Permissions{
   986  					{
   987  						Permission: "SELECT",
   988  					},
   989  					{
   990  						Permission: "DELETE",
   991  					},
   992  				},
   993  			},
   994  		},
   995  	}
   996  	return &sequelApi.DatabaseUser{
   997  		TypeMeta: gvkToTypeMeta(sequelApi.UserGVK),
   998  		ObjectMeta: metav1.ObjectMeta{
   999  			Name:      authserverName,
  1000  			Namespace: b.Name,
  1001  		},
  1002  		Spec: sequelApi.UserSpec{
  1003  			Type: sequelApi.CloudSAUserType,
  1004  			CommonOptions: sequelApi.CommonOptions{
  1005  				Prune: true,
  1006  				Force: true,
  1007  			},
  1008  			InstanceRef: sequelApi.InstanceReference{
  1009  				Name:      r.DatabaseName + dbInstance,
  1010  				ProjectID: r.ForemanProjectID,
  1011  			},
  1012  			ServiceAccount: &sequelApi.ServiceAccount{
  1013  				EmailRef:    fmt.Sprintf("%s.gserviceaccount.com", iamUsername),
  1014  				IAMUsername: iamUsername,
  1015  			},
  1016  			Grants: []sequelApi.Grant{
  1017  				grant,
  1018  			},
  1019  		},
  1020  	}
  1021  }
  1022  
  1023  func (r *BannerReconciler) createClusterctlDatabaseUser(b *bannerAPI.Banner) *sequelApi.DatabaseUser {
  1024  	hash := uuid.FromUUID(b.Status.ClusterInfraClusterEdgeID).Hash()
  1025  	clusterctlSAName := fmt.Sprintf("cctl-%s", hash)
  1026  	iamUsername := fmt.Sprintf("%s@%s.iam", clusterctlSAName, b.Spec.GCP.ProjectID)
  1027  
  1028  	grant := sequelApi.Grant{
  1029  		Schema: "public",
  1030  		TableGrant: []sequelApi.TableGrant{
  1031  			{
  1032  				Table: "clusters",
  1033  				Permissions: []sequelApi.Permissions{
  1034  					{
  1035  						Permission: "SELECT",
  1036  					},
  1037  					{
  1038  						Permission: "TRIGGER",
  1039  					},
  1040  					{
  1041  						Permission: "UPDATE",
  1042  					},
  1043  				},
  1044  			},
  1045  			{
  1046  				Table: "cluster_artifact_versions",
  1047  				Permissions: []sequelApi.Permissions{
  1048  					{
  1049  						Permission: "DELETE",
  1050  					},
  1051  					{
  1052  						Permission: "INSERT",
  1053  					},
  1054  					{
  1055  						Permission: "SELECT",
  1056  					},
  1057  				},
  1058  			},
  1059  			{
  1060  				Table: "cluster_config",
  1061  				Permissions: []sequelApi.Permissions{
  1062  					{
  1063  						Permission: "SELECT",
  1064  					},
  1065  				},
  1066  			},
  1067  			{
  1068  				Table: "id_provider_owner",
  1069  				Permissions: []sequelApi.Permissions{
  1070  					{
  1071  						Permission: "SELECT",
  1072  					},
  1073  				},
  1074  			},
  1075  			{
  1076  				Table: "id_provider_settings",
  1077  				Permissions: []sequelApi.Permissions{
  1078  					{
  1079  						Permission: "SELECT",
  1080  					},
  1081  				},
  1082  			},
  1083  			{
  1084  				Table: "banners",
  1085  				Permissions: []sequelApi.Permissions{
  1086  					{
  1087  						Permission: "SELECT",
  1088  					},
  1089  					{
  1090  						Permission: "UPDATE",
  1091  					},
  1092  				},
  1093  			},
  1094  			{
  1095  				Table: "cluster_bootstrap_tokens",
  1096  				Permissions: []sequelApi.Permissions{
  1097  					{
  1098  						Permission: "SELECT",
  1099  					},
  1100  					{
  1101  						Permission: "DELETE",
  1102  					},
  1103  				},
  1104  			},
  1105  			{
  1106  				Table: "terminals",
  1107  				Permissions: []sequelApi.Permissions{
  1108  					{
  1109  						Permission: "SELECT",
  1110  					},
  1111  				},
  1112  			},
  1113  			{
  1114  				Table: "artifact_registries",
  1115  				Permissions: []sequelApi.Permissions{
  1116  					{
  1117  						Permission: "SELECT",
  1118  					},
  1119  				},
  1120  			},
  1121  			{
  1122  				Table: "cluster_artifact_registries",
  1123  				Permissions: []sequelApi.Permissions{
  1124  					{
  1125  						Permission: "SELECT",
  1126  					},
  1127  				},
  1128  			},
  1129  			{
  1130  				Table: "helm_workloads",
  1131  				Permissions: []sequelApi.Permissions{
  1132  					{
  1133  						Permission: "SELECT",
  1134  					},
  1135  					{
  1136  						Permission: "DELETE",
  1137  					},
  1138  				},
  1139  			},
  1140  			{
  1141  				Table: "helm_secrets",
  1142  				Permissions: []sequelApi.Permissions{
  1143  					{
  1144  						Permission: "SELECT",
  1145  					},
  1146  				},
  1147  			},
  1148  			{
  1149  				Table: "helm_workload_config_maps",
  1150  				Permissions: []sequelApi.Permissions{
  1151  					{
  1152  						Permission: "SELECT",
  1153  					},
  1154  				},
  1155  			},
  1156  			{
  1157  				Table: "cluster_labels",
  1158  				Permissions: []sequelApi.Permissions{
  1159  					{
  1160  						Permission: "DELETE",
  1161  					},
  1162  					{
  1163  						Permission: "INSERT",
  1164  					},
  1165  					{
  1166  						Permission: "SELECT",
  1167  					},
  1168  				},
  1169  			},
  1170  			{
  1171  				Table: "log_replays",
  1172  				Permissions: []sequelApi.Permissions{
  1173  					{
  1174  						Permission: "SELECT",
  1175  					},
  1176  					{
  1177  						Permission: "UPDATE",
  1178  					},
  1179  				},
  1180  			},
  1181  			{
  1182  				Table: "helm_workload_labels",
  1183  				Permissions: []sequelApi.Permissions{
  1184  					{
  1185  						Permission: "SELECT",
  1186  					},
  1187  				},
  1188  			},
  1189  			{
  1190  				Table: "labels",
  1191  				Permissions: []sequelApi.Permissions{
  1192  					{
  1193  						Permission: "SELECT",
  1194  					},
  1195  				},
  1196  			},
  1197  			{
  1198  				Table: "watched_field_objects",
  1199  				Permissions: []sequelApi.Permissions{
  1200  					{
  1201  						Permission: "SELECT",
  1202  					},
  1203  				},
  1204  			},
  1205  			{
  1206  				Table: "watched_field_values",
  1207  				Permissions: []sequelApi.Permissions{
  1208  					{
  1209  						Permission: "SELECT",
  1210  					},
  1211  				},
  1212  			},
  1213  			{
  1214  				Table: "cluster_network_services",
  1215  				Permissions: []sequelApi.Permissions{
  1216  					{
  1217  						Permission: "SELECT",
  1218  					},
  1219  				},
  1220  			},
  1221  			{
  1222  				Table: "workload_cluster_mapping",
  1223  				Permissions: []sequelApi.Permissions{
  1224  					{
  1225  						Permission: "SELECT",
  1226  					},
  1227  				},
  1228  			},
  1229  			{
  1230  				Table: "capabilities",
  1231  				Permissions: []sequelApi.Permissions{
  1232  					{
  1233  						Permission: "SELECT",
  1234  					},
  1235  				},
  1236  			},
  1237  			{
  1238  				Table: "capabilities_to_banners",
  1239  				Permissions: []sequelApi.Permissions{
  1240  					{
  1241  						Permission: "SELECT",
  1242  					},
  1243  				},
  1244  			},
  1245  			{
  1246  				Table: "cluster_secret_leases",
  1247  				Permissions: []sequelApi.Permissions{
  1248  					{
  1249  						Permission: "INSERT",
  1250  					},
  1251  					{
  1252  						Permission: "SELECT",
  1253  					},
  1254  					{
  1255  						Permission: "UPDATE",
  1256  					},
  1257  				},
  1258  			},
  1259  			{
  1260  				Table: "cluster_secrets",
  1261  				Permissions: []sequelApi.Permissions{
  1262  					{
  1263  						Permission: "INSERT",
  1264  					},
  1265  					{
  1266  						Permission: "SELECT",
  1267  					},
  1268  					{
  1269  						Permission: "UPDATE",
  1270  					},
  1271  					{
  1272  						Permission: "DELETE",
  1273  					},
  1274  				},
  1275  			},
  1276  			{
  1277  				Table: "channels",
  1278  				Permissions: []sequelApi.Permissions{
  1279  					{
  1280  						Permission: "SELECT",
  1281  					},
  1282  				},
  1283  			},
  1284  			{
  1285  				Table: "helm_workloads_channels",
  1286  				Permissions: []sequelApi.Permissions{
  1287  					{
  1288  						Permission: "INSERT",
  1289  					},
  1290  					{
  1291  						Permission: "SELECT",
  1292  					},
  1293  				},
  1294  			},
  1295  		},
  1296  	}
  1297  
  1298  	return &sequelApi.DatabaseUser{
  1299  		TypeMeta: gvkToTypeMeta(sequelApi.UserGVK),
  1300  		ObjectMeta: metav1.ObjectMeta{
  1301  			Name:      clusterctlSAName,
  1302  			Namespace: b.Name,
  1303  		},
  1304  		Spec: sequelApi.UserSpec{
  1305  			Type: sequelApi.CloudSAUserType,
  1306  			CommonOptions: sequelApi.CommonOptions{
  1307  				Prune: true,
  1308  				Force: true,
  1309  			},
  1310  			InstanceRef: sequelApi.InstanceReference{
  1311  				Name:      r.DatabaseName + dbInstance,
  1312  				ProjectID: r.ForemanProjectID,
  1313  			},
  1314  			ServiceAccount: &sequelApi.ServiceAccount{
  1315  				EmailRef:    fmt.Sprintf("%s.gserviceaccount.com", iamUsername),
  1316  				IAMUsername: iamUsername,
  1317  			},
  1318  			Grants: []sequelApi.Grant{
  1319  				grant,
  1320  			},
  1321  		},
  1322  	}
  1323  }
  1324  
  1325  func createBannerWideNamespace(b *bannerAPI.Banner) (*syncedobjectApi.SyncedObject, error) {
  1326  	ns := corev1.Namespace{
  1327  		TypeMeta: metav1.TypeMeta{
  1328  			Kind:       "Namespace",
  1329  			APIVersion: "v1",
  1330  		},
  1331  		ObjectMeta: metav1.ObjectMeta{
  1332  			Name: b.Name,
  1333  		},
  1334  	}
  1335  	sobj, err := k8objectsutils.BuildSyncedObjectCoreV1(ns, b.Spec.GCP.ProjectID, "", b.Name, constants.BannerWideNamespace)
  1336  	if err != nil {
  1337  		return nil, err
  1338  	}
  1339  	return sobj, nil
  1340  }
  1341  
  1342  func (r *BannerReconciler) reconcilePlatformSecrets(ctx context.Context, b *bannerAPI.Banner) recerr.Error {
  1343  	log := ctrl.LoggerFrom(ctx).WithName("platform-secrets")
  1344  
  1345  	// create platform secrets when the secretmanager api is ready
  1346  	ready, err := r.checkAPI(ctx, b.Name, "secretmanager.googleapis.com")
  1347  	if err != nil {
  1348  		return recerr.New(err, bannerAPI.PlatformSecretsCreationFailedReason)
  1349  	}
  1350  	// return specific error indicating that the service isn't ready, so we requeue
  1351  	if !ready {
  1352  		err := fmt.Errorf("%w", ErrAPINotReady)
  1353  		return recerr.NewWait(err, bannerAPI.PlatformSecretsCreationFailedReason, r.DefaultRequeue)
  1354  	}
  1355  	err = r.copyPlatformSecrets(ctx, b.Spec.GCP.ProjectID)
  1356  	if err != nil {
  1357  		log.Error(err, "error copying platform secrets for project")
  1358  		return recerr.New(err, bannerAPI.PlatformSecretsCreationFailedReason)
  1359  	}
  1360  
  1361  	return nil
  1362  }
  1363  
  1364  // ownerRef creates an owner reference for this controller that should be added
  1365  // to objects this controller object owns. this enables things like
  1366  // automated garbage collection
  1367  func (r *BannerReconciler) ownerRef(b *bannerAPI.Banner) []metav1.OwnerReference {
  1368  	return []metav1.OwnerReference{
  1369  		*metav1.NewControllerRef(
  1370  			b,
  1371  			bannerAPI.GroupVersion.WithKind(reflect.TypeOf(bannerAPI.Banner{}).Name()),
  1372  		),
  1373  	}
  1374  }
  1375  
  1376  // createOpts returns client.CreatOptions marking this controller as the owner
  1377  // the result string should match what fluxcd/pkg/ssa.Onwer adds to resources
  1378  func (r *BannerReconciler) createOpts() *client.CreateOptions {
  1379  	return &client.CreateOptions{FieldManager: fmt.Sprintf("%s/%s", constants.Domain, r.Name)}
  1380  }
  1381  
  1382  func (r *BannerReconciler) setResourceManager() {
  1383  	if r.ResourceManager == nil {
  1384  		mgr := ssa.NewResourceManager(
  1385  			r.Client,
  1386  			polling.NewStatusPoller(r.Client, r.Client.RESTMapper(), polling.Options{}), ssa.Owner{Field: r.Name},
  1387  		)
  1388  		r.ResourceManager = mgr
  1389  	}
  1390  }
  1391  
  1392  func gvkToTypeMeta(gvk schema.GroupVersionKind) metav1.TypeMeta {
  1393  	v, k := gvk.ToAPIVersionAndKind()
  1394  	return metav1.TypeMeta{
  1395  		APIVersion: v,
  1396  		Kind:       k,
  1397  	}
  1398  }
  1399  
  1400  func (r *BannerReconciler) genForemanSO(obj client.Object, b *bannerAPI.Banner) *syncedobjectApi.SyncedObject {
  1401  	data, _ := json.Marshal(obj)
  1402  	data64 := base64.StdEncoding.EncodeToString(data)
  1403  	return &syncedobjectApi.SyncedObject{
  1404  		TypeMeta: metav1.TypeMeta{
  1405  			APIVersion: syncedobjectApi.GroupVersion.String(),
  1406  			Kind:       "SyncedObject",
  1407  		},
  1408  		ObjectMeta: metav1.ObjectMeta{
  1409  			Name:            obj.GetName(),
  1410  			Namespace:       b.Name,
  1411  			OwnerReferences: r.ownerRef(b),
  1412  		},
  1413  		Spec: syncedobjectApi.SyncedObjectSpec{
  1414  			Banner:  r.ForemanProjectID,
  1415  			Cluster: "foreman0",
  1416  			Object:  data64,
  1417  		},
  1418  	}
  1419  }
  1420  
  1421  func (r *BannerReconciler) createGARRepo(b *bannerAPI.Banner) *registryAPI.ArtifactRegistryRepository {
  1422  	desc := "Warehouse registry for K8s configuration packages"
  1423  
  1424  	return &registryAPI.ArtifactRegistryRepository{
  1425  		ObjectMeta: metav1.ObjectMeta{
  1426  			Name:      "warehouse",
  1427  			Namespace: b.Name,
  1428  			Annotations: map[string]string{
  1429  				kccmeta.DeletionPolicyAnnotation: kccmeta.DeletionPolicyAbandon,
  1430  			},
  1431  			OwnerReferences: r.ownerRef(b),
  1432  		},
  1433  		TypeMeta: gvkToTypeMeta(registryAPI.ArtifactRegistryRepositoryGVK),
  1434  		Spec: registryAPI.ArtifactRegistryRepositorySpec{
  1435  			Description: &desc,
  1436  			Format:      "DOCKER",
  1437  			Location:    r.GCPRegion,
  1438  		},
  1439  	}
  1440  }
  1441  
  1442  func (r *BannerReconciler) generateShipments(b *bannerAPI.Banner) ([]client.Object, error) {
  1443  	capabilities := generator.InfraCapabilities
  1444  	params := generator.BannerRenderParams{
  1445  		ClusterRenderParams: generator.ClusterRenderParams{
  1446  			ClusterType:              string(cluster.GKE),
  1447  			UUID:                     b.Name,
  1448  			Region:                   r.GCPRegion,
  1449  			Zone:                     r.GCPZone,
  1450  			ForemanGCPProjectID:      r.ForemanProjectID,
  1451  			GCPForemanProjectNumber:  r.GCPForemanProjectNumber,
  1452  			BannerID:                 b.Name,
  1453  			GCPProjectID:             b.Spec.GCP.ProjectID,
  1454  			BSLEUID:                  b.Spec.BSL.EnterpriseUnit.ID,
  1455  			BSLEdgeEnvPrefix:         r.BSLConfig.OrganizationPrefix,
  1456  			BSLEndpoint:              r.BSLConfig.Endpoint,
  1457  			BSLRootOrg:               r.BSLConfig.Root,
  1458  			Domain:                   r.Domain,
  1459  			DatasyncDNSName:          r.DatasyncDNSName,
  1460  			DatasyncDNSZone:          r.DatasyncDNSZone,
  1461  			DatabaseName:             r.DatabaseName,
  1462  			EdgeSecOptInCompliance:   r.EdgeSecOptInCompliance,
  1463  			EdgeSecMaxLeasePeriod:    r.EdgeSecMaxLeasePeriod,
  1464  			EdgeSecMaxValidityPeriod: r.EdgeSecMaxValidityPeriod,
  1465  		},
  1466  		PlatformInfraGCPProjectID: r.PlatInfraProjectID,
  1467  	}
  1468  
  1469  	shipmentOpts := &generator.ShipmentOpts{
  1470  		Prune:         true,
  1471  		Force:         true,
  1472  		Pallets:       []whv1.BaseArtifact{{Name: "couchdb-bannerinfra", Tag: "latest"}},
  1473  		Repository:    generator.GenerateShipmentRepo(r.GCPRegion, r.ForemanProjectID),
  1474  		Capabilities:  capabilities,
  1475  		Interval:      &metav1.Duration{Duration: 120 * time.Second},
  1476  		RetryInterval: &metav1.Duration{Duration: 20 * time.Second},
  1477  		Timeout:       &metav1.Duration{Duration: 90 * time.Second},
  1478  	}
  1479  	shipmentOpts.AddBannerRenderParams(params)
  1480  	shipment, err := shipmentOpts.BuildShipment(true, false)
  1481  	if err != nil {
  1482  		return nil, fmt.Errorf("unable to build couchdb-masters pallet: %w", err)
  1483  	}
  1484  	return []client.Object{shipment}, nil
  1485  }
  1486  

View as plain text