Source file src/edge-infra.dev/pkg/edge/controllers/bannerctl/bannerctl_test.go

Documentation: edge-infra.dev/pkg/edge/controllers/bannerctl

     1  package bannerctl
     2  
     3  import (
     4  	"context"
     5  	"database/sql"
     6  	"encoding/json"
     7  	"fmt"
     8  	"net/http"
     9  	"net/http/httptest"
    10  	"os"
    11  	"regexp"
    12  	"strings"
    13  	"testing"
    14  	"time"
    15  
    16  	metricsscope "cloud.google.com/go/monitoring/metricsscope/apiv1"
    17  	"cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
    18  	registryAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/artifactregistry/v1beta1"
    19  	computeAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/compute/v1beta1"
    20  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/iam/v1beta1"
    21  
    22  	k8sAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/k8s/v1alpha1"
    23  	loggingAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/logging/v1beta1"
    24  	resourceAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/resourcemanager/v1beta1"
    25  	serviceAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/serviceusage/v1beta1"
    26  	storageAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/storage/v1beta1"
    27  	"github.com/google/go-cmp/cmp"
    28  	"github.com/stretchr/testify/assert"
    29  	"github.com/stretchr/testify/suite"
    30  	"google.golang.org/api/option"
    31  	corev1 "k8s.io/api/core/v1"
    32  	"k8s.io/apimachinery/pkg/api/errors"
    33  	kmeta "k8s.io/apimachinery/pkg/api/meta"
    34  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    35  	"k8s.io/apimachinery/pkg/types"
    36  	ctrl "sigs.k8s.io/controller-runtime"
    37  	"sigs.k8s.io/controller-runtime/pkg/client"
    38  
    39  	"edge-infra.dev/pkg/edge/api/graph/model"
    40  	"edge-infra.dev/pkg/edge/api/testutils/seededpostgres"
    41  	bannerAPI "edge-infra.dev/pkg/edge/apis/banner/v1alpha1"
    42  	syncedobjectApi "edge-infra.dev/pkg/edge/apis/syncedobject/apis/v1alpha1"
    43  	capabilities "edge-infra.dev/pkg/edge/capabilities"
    44  	edgeconstants "edge-infra.dev/pkg/edge/constants"
    45  	bannerconstants "edge-infra.dev/pkg/edge/constants/api/banner"
    46  	"edge-infra.dev/pkg/edge/controllers/dbmetrics"
    47  	"edge-infra.dev/pkg/edge/controllers/util/edgedb"
    48  	"edge-infra.dev/pkg/edge/gcpinfra"
    49  	"edge-infra.dev/pkg/edge/registration"
    50  	"edge-infra.dev/pkg/f8n/gcp/k8s/controllers/dennis"
    51  	"edge-infra.dev/pkg/k8s/konfigkonnector/apis/meta"
    52  	"edge-infra.dev/pkg/k8s/meta/status"
    53  	"edge-infra.dev/pkg/k8s/runtime/controller"
    54  	"edge-infra.dev/pkg/k8s/runtime/inventory"
    55  	ff "edge-infra.dev/pkg/lib/featureflag"
    56  	fftest "edge-infra.dev/pkg/lib/featureflag/testutil"
    57  	"edge-infra.dev/pkg/lib/gcp/iam/roles"
    58  	gcpProject "edge-infra.dev/pkg/lib/gcp/project"
    59  	"edge-infra.dev/pkg/lib/logging"
    60  	"edge-infra.dev/pkg/lib/uuid"
    61  	"edge-infra.dev/test"
    62  	"edge-infra.dev/test/framework"
    63  	"edge-infra.dev/test/framework/gcp"
    64  	"edge-infra.dev/test/framework/integration"
    65  	"edge-infra.dev/test/framework/k8s"
    66  	"edge-infra.dev/test/framework/k8s/envtest"
    67  )
    68  
    69  const DefaultSecretValue = "mock secret value"
    70  
    71  var TestProjectNumber = "123456789000"
    72  
    73  func TestMain(m *testing.M) {
    74  	framework.HandleFlags()
    75  	// instantiate test logger
    76  	ctrl.SetLogger(logging.NewLogger().Logger)
    77  	os.Exit(m.Run())
    78  }
    79  
    80  type Suite struct {
    81  	*framework.Framework
    82  	*k8s.K8s
    83  	ctx       context.Context
    84  	timeout   time.Duration
    85  	tick      time.Duration
    86  	projectID string
    87  	sMgr      *mockSecretManager
    88  	msc       metricsScopesClient
    89  
    90  	db *sql.DB
    91  }
    92  
    93  type mockSecretManager struct {
    94  	clients map[string]*mockSecretManagerClient
    95  }
    96  
    97  func (sm *mockSecretManager) NewWithOptions(_ context.Context, projectID string, _ ...option.ClientOption) (secretManagerClient, error) {
    98  	if sm.clients[projectID] == nil {
    99  		sm.clients[projectID] = &mockSecretManagerClient{
   100  			secrets: make(map[string]*mockSecret),
   101  		}
   102  	}
   103  	return sm.clients[projectID], nil
   104  }
   105  
   106  type mockSecretManagerClient struct {
   107  	secrets map[string]*mockSecret
   108  }
   109  
   110  type mockSecret struct {
   111  	value []byte
   112  }
   113  
   114  // GetLatestSecretValue returns a secret value from the mock storage, or JIT creates it if missing. The controller
   115  // under test reads from one Secret Manager and writes to another, but this mock only has one storage location. So
   116  // the first read is expected to miss
   117  func (smc *mockSecretManagerClient) GetLatestSecretValue(_ context.Context, secretID string) ([]byte, error) {
   118  	s := smc.secrets[secretID]
   119  	if s == nil {
   120  		return nil, fmt.Errorf("secret %s not found", secretID)
   121  	}
   122  	return smc.secrets[secretID].value, nil
   123  }
   124  
   125  // GetLatestSecretValueInfo mocks base method.
   126  func (smc *mockSecretManagerClient) GetLatestSecretValueInfo(_ context.Context, secretID string) (*secretmanagerpb.SecretVersion, error) {
   127  	s := smc.secrets[secretID]
   128  	if s == nil {
   129  		return nil, fmt.Errorf("secret %s not found", secretID)
   130  	}
   131  	return &secretmanagerpb.SecretVersion{
   132  		Name: secretID,
   133  	}, nil
   134  }
   135  
   136  func (smc *mockSecretManagerClient) GetSecret(_ context.Context, secretID string) (*secretmanagerpb.Secret, error) {
   137  	s := smc.secrets[secretID]
   138  	if s == nil {
   139  		return nil, fmt.Errorf("secret %s not found", secretID)
   140  	}
   141  	spb := &secretmanagerpb.Secret{
   142  		Name: secretID,
   143  	}
   144  	return spb, nil
   145  }
   146  
   147  func (smc *mockSecretManagerClient) AddSecret(_ context.Context, secretID string, _ []byte, _ map[string]string, _ bool, _ *time.Time, _ string) error {
   148  	smc.secrets[secretID] = &mockSecret{value: []byte(DefaultSecretValue)}
   149  	return nil
   150  }
   151  
   152  type mockMetricsScopeClient struct{}
   153  
   154  func (c *mockMetricsScopeClient) AddMonitoredProject(_ context.Context, _ string) (*metricsscope.CreateMonitoredProjectOperation, error) {
   155  	return &metricsscope.CreateMonitoredProjectOperation{}, nil
   156  }
   157  
   158  func falsifyResourceReadiness(conds []k8sAPI.Condition) []k8sAPI.Condition {
   159  	hasCond := false
   160  	for _, c := range conds {
   161  		if c.Type == "Ready" {
   162  			hasCond = true
   163  		}
   164  	}
   165  	if !hasCond {
   166  		conds = append(conds, k8sAPI.Condition{Type: "Ready", Status: "True"})
   167  	}
   168  	return conds
   169  }
   170  
   171  func isOwnedByBanner(obj client.Object, banner string) bool {
   172  	for _, o := range obj.GetOwnerReferences() {
   173  		if o.Name == banner {
   174  			return true
   175  		}
   176  	}
   177  	return false
   178  }
   179  
   180  func TestBannerController(t *testing.T) {
   181  	testEnv := envtest.Setup()
   182  	cfg, opts := controller.ProcessOptions(controller.WithCfg(testEnv.Config), controller.WithMetricsAddress("0"))
   183  	opts.Scheme = createScheme()
   184  	mgr, err := ctrl.NewManager(cfg, opts)
   185  	test.NoError(err)
   186  
   187  	// mock feature flags
   188  	fftest.MustInitTestFeatureFlagsForContexts([]fftest.TestFlag{
   189  		{
   190  			FlagContext: ff.NewEnvironmentContext(),
   191  			Name:        ff.UseBannerCTLPruning,
   192  			Value:       true,
   193  		},
   194  	})
   195  
   196  	f := framework.New("bannerctl").Component("bannerctl")
   197  
   198  	sp, err := seededpostgres.New()
   199  	test.NoError(err)
   200  	defer sp.Close()
   201  
   202  	db, err := sp.DB()
   203  	test.NoError(err)
   204  	defer db.Close()
   205  
   206  	// Create Suite with base values
   207  	s := &Suite{
   208  		Framework: f,
   209  		ctx:       context.Background(),
   210  		timeout:   10 * time.Second,
   211  		tick:      50 * time.Millisecond,
   212  		db:        db,
   213  	}
   214  	// Update Suite based on integration test config
   215  	// and initialize mock GCP services if necessary
   216  	if integration.IsIntegrationTest() {
   217  		s.projectID = gcp.GCloud.ProjectID
   218  	} else {
   219  		s.projectID = "mock gcp projectid"
   220  		s.msc = &mockMetricsScopeClient{}
   221  	}
   222  	s.sMgr = &mockSecretManager{clients: make(map[string]*mockSecretManagerClient)}
   223  	for _, secretID := range edgeconstants.PlatformSecretIDs {
   224  		c, _ := s.sMgr.NewWithOptions(s.ctx, s.projectID)
   225  		_ = c.AddSecret(s.ctx, secretID, []byte("mock secret value"), nil, false, nil, "")
   226  	}
   227  
   228  	totpSecret := "totp-secret"
   229  	srv := httptest.NewServer(http.HandlerFunc(
   230  		registration.GraphQLHandler(assert.New(t),
   231  			registration.WithTotpSecret(totpSecret),
   232  			registration.WithExistingCluster("couchdb-enabled"),
   233  		)))
   234  	defer srv.Close()
   235  
   236  	bslSrv, bslClient := FakeBslClient()
   237  	defer bslSrv.Close()
   238  
   239  	dbm := dbmetrics.New("bannerctl")
   240  
   241  	reconciler := &BannerReconciler{
   242  		Client:                   mgr.GetClient(),
   243  		Log:                      ctrl.Log.WithName("bannerctl"),
   244  		ForemanProjectID:         s.projectID,
   245  		PlatInfraProjectID:       s.projectID,
   246  		SecretManager:            s.sMgr,
   247  		MetricsScopesClient:      s.msc,
   248  		DefaultRequeue:           10 * time.Millisecond,
   249  		Name:                     "bannerctl",
   250  		EdgeAPI:                  srv.URL,
   251  		TotpSecret:               totpSecret,
   252  		Domain:                   "dev0.edge-preprod.dev",
   253  		DatasyncDNSName:          datasyncDNSName,
   254  		DatasyncDNSZone:          datasyncDNSZone,
   255  		DatabaseName:             databaseName,
   256  		Conditions:               bannerConditions,
   257  		EdgeDB:                   &edgedb.EdgeDB{DB: db},
   258  		Recorder:                 dbm,
   259  		BSLClient:                bslClient,
   260  		GCPRegion:                gcpRegion,
   261  		GCPZone:                  gcpZone,
   262  		GCPForemanProjectNumber:  gcpForemanProjectNumber,
   263  		BSLConfig:                bslConfig,
   264  		EdgeSecOptInCompliance:   true,
   265  		EdgeSecMaxLeasePeriod:    "48h",
   266  		EdgeSecMaxValidityPeriod: "60d",
   267  	}
   268  	err = reconciler.SetupWithManager(mgr)
   269  	test.NoError(err)
   270  
   271  	k := k8s.New(testEnv.Config, k8s.WithCtrlManager(mgr), k8s.WithKonfigKonnector())
   272  	s.K8s = k
   273  	f.Register(k)
   274  
   275  	suite.Run(t, s)
   276  
   277  	t.Cleanup(func() {
   278  		f.NoError(testEnv.Stop())
   279  	})
   280  }
   281  
   282  func (s *Suite) TestBannerCreation_SecondaryResources() {
   283  	bannerGUID := uuid.New().UUID
   284  	generatedProjID := fmt.Sprintf("%s-%s", gcpinfra.ProjectIDPrefix, gcpProject.RandAN(29-(len(gcpinfra.ProjectIDPrefix))))
   285  	testBSLID := uuid.New().UUID
   286  	testOrg := uuid.New().UUID
   287  	banner := &bannerAPI.Banner{
   288  		ObjectMeta: metav1.ObjectMeta{
   289  			Name: bannerGUID,
   290  		},
   291  		Spec: bannerAPI.BannerSpec{
   292  			DisplayName: "testy-mcbanner",
   293  			GCP: bannerAPI.GCPConfig{
   294  				ProjectID: generatedProjID,
   295  			},
   296  			BSL: bannerAPI.BSLConfig{
   297  				EnterpriseUnit: bannerAPI.BSLEnterpriseUnit{
   298  					ID: testBSLID,
   299  				},
   300  				Organization: bannerAPI.BSLOrganization{
   301  					Name: testOrg,
   302  				},
   303  			},
   304  			Enablements: []string{CouchDBEnablement},
   305  		},
   306  	}
   307  
   308  	s.createBanner(banner)
   309  	defer s.deleteBanner(banner)
   310  
   311  	// Secondary resources that should be created for a Banner:
   312  	// GCP Project KCC resource is created and added to Banner status
   313  	project := &resourceAPI.Project{
   314  		ObjectMeta: metav1.ObjectMeta{
   315  			Name:      projectName,
   316  			Namespace: bannerGUID,
   317  		},
   318  	}
   319  	s.Require().Eventually(func() bool {
   320  		err := s.Client.Get(s.ctx, types.NamespacedName{
   321  			Name:      project.Name,
   322  			Namespace: project.Namespace,
   323  		}, project)
   324  		return err == nil
   325  	}, s.timeout, s.tick, "expected Project was never found")
   326  	s.Require().True(isOwnedByBanner(project, bannerGUID), "expected (namespaced) Project to be owned by (cluster scoped) Banner")
   327  	// there is no Project controller to do this for us. simulate a ready resource
   328  	project.Status.Conditions = falsifyResourceReadiness(project.Status.Conditions)
   329  	project.Status.Number = &TestProjectNumber
   330  	s.Require().NoError(s.Client.Update(s.ctx, project))
   331  	s.Require().Eventually(func() bool {
   332  		err := s.Client.Get(s.ctx, types.NamespacedName{
   333  			Name:      banner.Name,
   334  			Namespace: banner.Namespace,
   335  		}, banner)
   336  		return err == nil && banner.Status.ProjectRef != ""
   337  	}, s.timeout, s.tick, "expected Banner Status was never set")
   338  
   339  	// Verify GCP Project was created and matches Banner status
   340  	projectViaStatusRef := &resourceAPI.Project{}
   341  	s.Require().Eventually(func() bool {
   342  		err := s.Client.Get(s.ctx, types.NamespacedName{
   343  			Name:      strings.Split(banner.Status.ProjectRef, "/")[1],
   344  			Namespace: strings.Split(banner.Status.ProjectRef, "/")[0],
   345  		}, projectViaStatusRef)
   346  		return err == nil
   347  	}, s.timeout, s.tick, "expected Project was never found")
   348  	s.Equal(projectViaStatusRef, project, "could not find Project obj via Banner.Status.ProjectRef")
   349  
   350  	// K8s Namespace with correct project annotation
   351  	namespace := &corev1.Namespace{}
   352  	s.Require().Eventually(func() bool {
   353  		err := s.Client.Get(s.ctx, types.NamespacedName{
   354  			Name: bannerGUID,
   355  		}, namespace)
   356  		if err != nil {
   357  			return false
   358  		}
   359  		nsAnnos := namespace.Annotations
   360  		if nsAnnos != nil {
   361  			return s.Equal(generatedProjID, namespace.Annotations[meta.ProjectAnnotation]) && s.Equal(meta.DeletionPolicyAbandon, namespace.Annotations[meta.DeletionPolicyAnnotation])
   362  		}
   363  		return false
   364  	}, s.timeout, s.tick, "expected Namespace was never found or didnt have correct annotation")
   365  
   366  	//cluster := &edgeCluster.Cluster{}
   367  	//s.Require().Eventually(func() bool {
   368  	//	err := s.Client.Get(s.ctx, types.NamespacedName{
   369  	//		Name: bannerconstants.CreateBannerClusterInfraName(banner.Spec.DisplayName),
   370  	//	}, cluster)
   371  	//	return err == nil
   372  	//}, s.timeout, s.tick, "expected edge cluster was never found")
   373  	//s.Require().True(isOwnedByBanner(cluster, bannerGUID), "expected edge cluster to be owned by Banner")
   374  
   375  	// APIs (Service object)
   376  	s.Require().Eventually(func() bool {
   377  		for _, api := range gcpinfra.TenantAPIs {
   378  			service := &serviceAPI.Service{}
   379  			err := s.Client.Get(s.ctx, types.NamespacedName{
   380  				Name:      api,
   381  				Namespace: bannerGUID,
   382  			}, service)
   383  			if errors.IsNotFound(err) {
   384  				return false
   385  			}
   386  			s.Require().True(isOwnedByBanner(service, bannerGUID), "expected Service to be owned by Banner")
   387  			s.Require().Equal(meta.DeletionPolicyAbandon, service.Annotations[meta.DeletionPolicyAnnotation])
   388  			// if the service was created, force it to be ready, it wont ever be ready in the test env
   389  			// TODO: integration
   390  			service.Status.Conditions = falsifyResourceReadiness(service.Status.Conditions)
   391  			s.Require().NoError(s.Client.Update(s.ctx, service))
   392  		}
   393  		return true
   394  	}, s.timeout, s.tick, "not all Services were created (enabled)")
   395  
   396  	// StorageBucket and roles
   397  	sb := &storageAPI.StorageBucket{}
   398  	s.Require().Eventually(func() bool {
   399  		err := s.Client.Get(s.ctx, types.NamespacedName{
   400  			Name:      generatedProjID,
   401  			Namespace: bannerGUID,
   402  		}, sb)
   403  		return err == nil
   404  	}, s.timeout, s.tick, "expected StorageBucket was never found")
   405  	s.Require().True(isOwnedByBanner(sb, bannerGUID), "expected StorageBucket to be owned by Banner")
   406  	s.Require().Equal(meta.DeletionPolicyAbandon, sb.Annotations[meta.DeletionPolicyAnnotation])
   407  
   408  	// Integration? platform secrets and metrics scope from real gcp
   409  	if integration.IsIntegrationTest() {
   410  		// TODO: Assert real values from integration test secretmanager. The expected value
   411  		// should come from test project values instead of hardcoded mocks
   412  		for _, secretID := range edgeconstants.PlatformSecretIDs {
   413  			v, err := s.sMgr.clients[bannerGUID].GetLatestSecretValue(s.ctx, secretID)
   414  			s.NoError(err)
   415  			s.Equal(DefaultSecretValue, string(v))
   416  		}
   417  	} else {
   418  		// Platform secrets
   419  		s.Require().Eventually(func() bool {
   420  			c := s.sMgr.clients[generatedProjID]
   421  			if c == nil {
   422  				return false
   423  			}
   424  			for _, secretID := range edgeconstants.PlatformSecretIDs {
   425  				v, err := c.GetLatestSecretValue(s.ctx, secretID)
   426  				if err != nil {
   427  					return false
   428  				}
   429  				if !s.Equal(DefaultSecretValue, string(v)) {
   430  					return false
   431  				}
   432  			}
   433  			return true
   434  		}, s.timeout, s.tick, "never got a generated GCP Project ID")
   435  	}
   436  
   437  	// Remote access ComputeAddress
   438  	cAddr := &computeAPI.ComputeAddress{
   439  		ObjectMeta: metav1.ObjectMeta{
   440  			Name:      fmt.Sprintf("%s-%s", bannerconstants.RemoteAccessIPName, banner.Name),
   441  			Namespace: bannerGUID,
   442  		},
   443  	}
   444  	s.Require().Eventually(func() bool {
   445  		err := s.Client.Get(s.ctx, types.NamespacedName{
   446  			Name:      cAddr.Name,
   447  			Namespace: cAddr.Namespace,
   448  		}, cAddr)
   449  		return err == nil
   450  	}, s.timeout, s.tick, fmt.Sprintf("expected ComputeAddress %s/%s was never found", cAddr.Namespace, cAddr.Name))
   451  	dnsRecordCfgs := cAddr.Annotations[dennis.RecordConfigsAnnotation]
   452  	s.Require().NotEmpty(dnsRecordCfgs, fmt.Sprintf("ComputeAddress was missing required annotation %s", dennis.RecordConfigsAnnotation))
   453  	cfgs := &[]dennis.RecordConfig{}
   454  	err := json.Unmarshal([]byte(dnsRecordCfgs), cfgs)
   455  	s.Require().NoError(err, fmt.Sprintf("ComputeAddress had invalid %s annotation", dennis.RecordConfigsAnnotation))
   456  	s.Require().Equal(meta.DeletionPolicyAbandon, cAddr.Annotations[meta.DeletionPolicyAnnotation])
   457  
   458  	// source: https://www.socketloop.com/tutorials/golang-use-regular-expression-to-validate-domain-name with modifications
   459  	// for trailing '.' per RFC 1034 https://datatracker.ietf.org/doc/html/rfc1034
   460  	re := regexp.MustCompile(`((([a-zA-Z]{1})|([a-zA-Z]{1}[a-zA-Z]{1})|([a-zA-Z]{1}[0-9]{1})|([0-9]{1}[a-zA-Z]{1})|([a-zA-Z0-9][a-zA-Z0-9-_]{1,61}[a-zA-Z0-9]))\.){2}([a-zA-Z]{2,6}|([a-zA-Z0-9-]){2,30}\.[a-zA-Z]{2,3})\.$`)
   461  	for _, cfg := range *cfgs {
   462  		s.Require().True(re.MatchString(cfg.Name), fmt.Sprintf("remote access dns name '%s' was not valid", cfg.Name))
   463  	}
   464  
   465  	// LoggingLogSink
   466  	lls := &loggingAPI.LoggingLogSink{}
   467  	s.Require().Eventually(func() bool {
   468  		err := s.Client.Get(s.ctx, types.NamespacedName{
   469  			Name:      logSinkName,
   470  			Namespace: bannerGUID,
   471  		}, lls)
   472  		return err == nil
   473  	}, s.timeout, s.tick, "expected LoggingLogSink was never found")
   474  	s.Require().True(isOwnedByBanner(lls, bannerGUID), "expected LoggingLogSink to be owned by Banner")
   475  	s.Require().Equal(fmt.Sprintf("storage.googleapis.com/%s-siem", s.projectID), lls.Spec.Destination.StorageBucketRef.External)
   476  	s.Require().Equal(banner.Spec.GCP.ProjectID, lls.Spec.ProjectRef.External)
   477  	s.Require().True(*lls.Spec.UniqueWriterIdentity, "expected LoggingLogSink UniqueWriterIdentity to be true")
   478  
   479  	// Metrics scope is a NOP. Just dont error here
   480  
   481  	// Banner eventually becomes ready
   482  	s.Require().Eventually(func() bool {
   483  		err := s.Client.Get(s.ctx, types.NamespacedName{
   484  			Name:      banner.Name,
   485  			Namespace: banner.Namespace,
   486  		}, banner)
   487  		return err == nil && kmeta.IsStatusConditionTrue(banner.Status.Conditions, status.ReadyCondition)
   488  	}, s.timeout, s.tick, "Banner never became Ready")
   489  
   490  	namespaceSO := &syncedobjectApi.SyncedObject{}
   491  	s.Require().Eventually(func() bool {
   492  		err := s.Client.Get(s.ctx, types.NamespacedName{
   493  			Name:      edgeconstants.BannerWideNamespace,
   494  			Namespace: banner.Name,
   495  		}, namespaceSO)
   496  		return err == nil
   497  	}, s.timeout, s.tick, "expected namespace synced obj was never found")
   498  	s.Equal(namespaceSO.Spec.Cluster, "", "namespace synced obj does not have expected empty cluster")
   499  	s.Equal(namespaceSO.Spec.Banner, banner.Spec.GCP.ProjectID, "namespace synced obj does not have expected project")
   500  
   501  	// Foreman0 synced objects
   502  	expectedForemanSO := []*syncedobjectApi.SyncedObject{
   503  		// 1. Emissary Mapping (banner ingress proxy)
   504  		{
   505  			ObjectMeta: metav1.ObjectMeta{
   506  				Name:      fmt.Sprintf("banner-proxy-%s", banner.Name),
   507  				Namespace: banner.Name,
   508  			},
   509  		},
   510  	}
   511  
   512  	// Validate all foreman0 SO's get created with correct foreman0 name and project id
   513  	for _, so := range expectedForemanSO {
   514  		s.pollForSyncedObject(so)
   515  		s.Require().Equal(s.projectID, so.Spec.Banner, fmt.Sprintf("SyncedObject %s/%s does not have banner set", so.Name, so.Namespace))
   516  		s.Require().Equal("foreman0", so.Spec.Cluster, fmt.Sprintf("synced object %s/%s does not have cluster set", so.Name, so.Namespace))
   517  	}
   518  
   519  	// Checking GCP infra for KCC last since it depends on the cluster-infra cluster's EdgeID
   520  	var hash string
   521  	var kccResourceName string
   522  
   523  	infraIAMSA := &v1beta1.IAMServiceAccount{}
   524  	s.Require().Eventually(func() bool {
   525  		if banner.Status.ClusterInfraClusterEdgeID != "" {
   526  			hash = uuid.FromUUID(banner.Status.ClusterInfraClusterEdgeID).Hash()
   527  			kccResourceName = fmt.Sprintf("kcc-%s", hash)
   528  		}
   529  		err := s.Client.Get(s.ctx, types.NamespacedName{
   530  			Name:      kccResourceName,
   531  			Namespace: bannerGUID,
   532  		}, infraIAMSA)
   533  		return err == nil
   534  	}, s.timeout, s.tick, "expected kcc IAMServiceAccount was never found")
   535  	s.Require().True(isOwnedByBanner(infraIAMSA, bannerGUID), "expected IAMServiceAccount to be owned by Banner")
   536  	s.Require().Equal(meta.DeletionPolicyAbandon, infraIAMSA.Annotations[meta.DeletionPolicyAnnotation])
   537  	s.Require().Equal(fmt.Sprintf("k8s cfg connector for project %s", banner.Spec.DisplayName), *infraIAMSA.Spec.DisplayName)
   538  
   539  	projectOwner := &v1beta1.IAMPolicyMember{}
   540  	s.Require().Eventually(func() bool {
   541  		err := s.Client.Get(s.ctx, types.NamespacedName{
   542  			Name:      fmt.Sprintf("%s-owner", kccResourceName),
   543  			Namespace: bannerGUID,
   544  		}, projectOwner)
   545  		return err == nil
   546  	}, s.timeout, s.tick, "expected kcc project owner IAMPolicyMember was never found")
   547  	s.Require().True(isOwnedByBanner(projectOwner, bannerGUID), "expected IAMPolicyMember to be owned by Banner")
   548  	s.Require().Equal(meta.DeletionPolicyAbandon, projectOwner.Annotations[meta.DeletionPolicyAnnotation])
   549  	s.Require().Equal(fmt.Sprintf("serviceAccount:%s@%s.iam.gserviceaccount.com", kccResourceName, generatedProjID), *projectOwner.Spec.Member)
   550  	s.Require().Equal(resourceAPI.SchemeGroupVersion.String(), projectOwner.Spec.ResourceRef.APIVersion)
   551  	s.Require().Equal(resourceAPI.ProjectGVK.Kind, projectOwner.Spec.ResourceRef.Kind)
   552  	s.Require().Equal(generatedProjID, projectOwner.Spec.ResourceRef.External)
   553  	s.Require().Equal(roles.Owner, projectOwner.Spec.Role)
   554  
   555  	logWriter := &v1beta1.IAMPolicyMember{}
   556  	policyName := fmt.Sprintf("%s-logging-logwriter", kccResourceName)
   557  	s.Require().Eventually(func() bool {
   558  		err := s.Client.Get(s.ctx, types.NamespacedName{
   559  			Name:      policyName,
   560  			Namespace: bannerGUID,
   561  		}, logWriter)
   562  		return err == nil
   563  	}, s.timeout, s.tick, "expected kcc project owner IAMPolicyMember was never found")
   564  	s.Require().True(isOwnedByBanner(logWriter, bannerGUID), "expected IAMPolicyMember to be owned by Banner")
   565  	s.Require().Equal(meta.DeletionPolicyAbandon, logWriter.Annotations[meta.DeletionPolicyAnnotation])
   566  	s.Require().Equal(fmt.Sprintf("serviceAccount:%s@%s.iam.gserviceaccount.com", kccResourceName, generatedProjID), *logWriter.Spec.Member)
   567  	s.Require().Equal(resourceAPI.SchemeGroupVersion.String(), logWriter.Spec.ResourceRef.APIVersion)
   568  	s.Require().Equal(resourceAPI.ProjectGVK.Kind, logWriter.Spec.ResourceRef.Kind)
   569  	s.Require().Equal(s.projectID, logWriter.Spec.ResourceRef.External)
   570  	s.Require().Equal(roles.ProjectAdmin, logWriter.Spec.Role)
   571  	s.Require().Equal("kcc_delegated_foreman_roles", logWriter.Spec.Condition.Title)
   572  	s.Require().Equal("KCC delegated roles on foreman", *logWriter.Spec.Condition.Description)
   573  	s.Require().Equal(KccDelegatedRolesExpression, logWriter.Spec.Condition.Expression)
   574  
   575  	pubSubAdmin := &v1beta1.IAMPolicyMember{}
   576  	policyName = fmt.Sprintf("%s-pubsub-admin", kccResourceName)
   577  	s.Require().Eventually(func() bool {
   578  		err := s.Client.Get(s.ctx, types.NamespacedName{
   579  			Name:      policyName,
   580  			Namespace: bannerGUID,
   581  		}, pubSubAdmin)
   582  		return err == nil
   583  	}, s.timeout, s.tick, "expected kcc project owner IAMPolicyMember was never found")
   584  	s.Require().True(isOwnedByBanner(pubSubAdmin, bannerGUID), "expected IAMPolicyMember to be owned by Banner")
   585  	s.Require().Equal(meta.DeletionPolicyAbandon, pubSubAdmin.Annotations[meta.DeletionPolicyAnnotation])
   586  	s.Require().Equal(fmt.Sprintf("serviceAccount:%s@%s.iam.gserviceaccount.com", kccResourceName, generatedProjID), *pubSubAdmin.Spec.Member)
   587  	s.Require().Equal(resourceAPI.SchemeGroupVersion.String(), pubSubAdmin.Spec.ResourceRef.APIVersion)
   588  	s.Require().Equal(resourceAPI.ProjectGVK.Kind, pubSubAdmin.Spec.ResourceRef.Kind)
   589  	s.Require().Equal(s.projectID, pubSubAdmin.Spec.ResourceRef.External)
   590  	s.Require().Equal(roles.PubsubAdmin, pubSubAdmin.Spec.Role)
   591  	s.Require().Nil(pubSubAdmin.Spec.Condition)
   592  
   593  	wiMember := &v1beta1.IAMPolicyMember{}
   594  	s.Require().Eventually(func() bool {
   595  		err := s.Client.Get(s.ctx, types.NamespacedName{
   596  			Name:      fmt.Sprintf("%s-workload-id", kccResourceName),
   597  			Namespace: bannerGUID,
   598  		}, wiMember)
   599  		return err == nil
   600  	}, s.timeout, s.tick, "expected kcc workload identity IAMPolicyMember was never found")
   601  	s.Require().True(isOwnedByBanner(wiMember, bannerGUID), "expected IAMPolicyMember to be owned by Banner")
   602  	s.Require().Equal(meta.DeletionPolicyAbandon, wiMember.Annotations[meta.DeletionPolicyAnnotation])
   603  	s.Require().Equal(fmt.Sprintf("serviceAccount:%s.svc.id.goog[cnrm-system/cnrm-controller-manager]", generatedProjID), *wiMember.Spec.Member)
   604  	s.Require().Equal(v1beta1.SchemeGroupVersion.String(), wiMember.Spec.ResourceRef.APIVersion)
   605  	s.Require().Equal(v1beta1.IAMServiceAccountGVK.Kind, wiMember.Spec.ResourceRef.Kind)
   606  	s.Require().Equal(kccResourceName, wiMember.Spec.ResourceRef.Name)
   607  	s.Require().Equal(roles.WorkloadIdentityUser, wiMember.Spec.Role)
   608  
   609  	s.validateEdgeLabels(banner)
   610  	s.checkInfraStatusDatabaseValue(banner, edgedb.InfraStatusReady)
   611  }
   612  
   613  func (s *Suite) TestBannerCreation_UsingWarehouse() {
   614  	bannerGUID := uuid.New().UUID
   615  	generatedProjID := fmt.Sprintf("%s-%s", gcpinfra.ProjectIDPrefix, gcpProject.RandAN(29-(len(gcpinfra.ProjectIDPrefix))))
   616  	banner := &bannerAPI.Banner{
   617  		ObjectMeta: metav1.ObjectMeta{
   618  			Name: bannerGUID,
   619  		},
   620  		Spec: bannerAPI.BannerSpec{
   621  			DisplayName: "dev1-banner-use-warehouse",
   622  			GCP: bannerAPI.GCPConfig{
   623  				ProjectID: generatedProjID,
   624  			},
   625  			BSL: bannerAPI.BSLConfig{
   626  				EnterpriseUnit: bannerAPI.BSLEnterpriseUnit{
   627  					ID: uuid.New().UUID,
   628  				},
   629  				Organization: bannerAPI.BSLOrganization{
   630  					Name: "test-org-dev1",
   631  				},
   632  			},
   633  		},
   634  	}
   635  
   636  	s.createBanner(banner)
   637  	defer s.deleteBanner(banner)
   638  
   639  	// 0.1 GCP Project KCC resource is created and added to Banner status, becomes ready
   640  	project := &resourceAPI.Project{
   641  		ObjectMeta: metav1.ObjectMeta{
   642  			Name:      projectName,
   643  			Namespace: bannerGUID,
   644  		},
   645  	}
   646  	s.Require().Eventually(func() bool {
   647  		err := s.Client.Get(s.ctx, types.NamespacedName{
   648  			Name:      project.Name,
   649  			Namespace: project.Namespace,
   650  		}, project)
   651  		return err == nil
   652  	}, s.timeout, s.tick, "expected Project was never found")
   653  	s.Require().True(isOwnedByBanner(project, bannerGUID), "expected (namespaced) Project to be owned by (cluster scoped) Banner")
   654  	// there is no Project controller to do this for us. simulate a ready resource
   655  	project.Status.Conditions = falsifyResourceReadiness(project.Status.Conditions)
   656  	project.Status.Number = &TestProjectNumber
   657  	s.Require().NoError(s.Client.Update(s.ctx, project))
   658  
   659  	// 1. Warehouse Service dependency
   660  	s.Require().Eventually(func() bool {
   661  		service := &serviceAPI.Service{}
   662  		err := s.Client.Get(s.ctx, types.NamespacedName{
   663  			Name:      "artifactregistry.googleapis.com",
   664  			Namespace: bannerGUID,
   665  		}, service)
   666  		if errors.IsNotFound(err) {
   667  			return false
   668  		}
   669  		s.Require().True(isOwnedByBanner(service, bannerGUID), "expected Service to be owned by Banner")
   670  		s.Require().Equal(meta.DeletionPolicyAbandon, service.Annotations[meta.DeletionPolicyAnnotation])
   671  		// if the service was created, force it to be ready, it wont ever be ready in the test env
   672  		service.Status.Conditions = falsifyResourceReadiness(service.Status.Conditions)
   673  		s.Require().NoError(s.Client.Update(s.ctx, service))
   674  		return true
   675  	}, s.timeout, s.tick, "not all Services were created (enabled)")
   676  
   677  	// 2. Warehouse Registry Repository
   678  	garRepo := &registryAPI.ArtifactRegistryRepository{}
   679  	s.Require().Eventually(func() bool {
   680  		err := s.Client.Get(s.ctx, types.NamespacedName{
   681  			Name:      "warehouse",
   682  			Namespace: bannerGUID,
   683  		}, garRepo)
   684  		return err == nil
   685  	}, s.timeout, s.tick, "expected ArtifactRegistryRepository was never found")
   686  	s.Require().Equal(meta.DeletionPolicyAbandon, garRepo.Annotations[meta.DeletionPolicyAnnotation])
   687  }
   688  
   689  func (s *Suite) pollForSyncedObject(so *syncedobjectApi.SyncedObject) {
   690  	s.Require().Eventually(func() bool {
   691  		err := s.Client.Get(s.ctx, types.NamespacedName{
   692  			Name:      so.Name,
   693  			Namespace: so.Namespace,
   694  		}, so)
   695  		return err == nil
   696  	}, s.timeout, s.tick, fmt.Sprintf("expected SyncedObject %s/%s was never found", so.Namespace, so.Name))
   697  }
   698  
   699  func (s *Suite) TestBannerCreation_WithInventoryPruning() {
   700  	bannerGUID := uuid.New().UUID
   701  	generatedProjID := fmt.Sprintf("%s-%s", gcpinfra.ProjectIDPrefix, gcpProject.RandAN(29-(len(gcpinfra.ProjectIDPrefix))))
   702  	banner := &bannerAPI.Banner{
   703  		ObjectMeta: metav1.ObjectMeta{
   704  			Name: bannerGUID,
   705  		},
   706  		Spec: bannerAPI.BannerSpec{
   707  			DisplayName: "test-banner-prune",
   708  			GCP: bannerAPI.GCPConfig{
   709  				ProjectID: generatedProjID,
   710  			},
   711  			BSL: bannerAPI.BSLConfig{
   712  				EnterpriseUnit: bannerAPI.BSLEnterpriseUnit{
   713  					ID: uuid.New().UUID,
   714  				},
   715  				Organization: bannerAPI.BSLOrganization{
   716  					Name: "test-org-dev-2",
   717  				},
   718  			},
   719  		},
   720  	}
   721  
   722  	s.createBanner(banner)
   723  	defer s.deleteBanner(banner)
   724  
   725  	// Secondary resources that should be created for a Banner:
   726  	// GCP Project KCC resource is created and added to Banner status
   727  	project := &resourceAPI.Project{
   728  		ObjectMeta: metav1.ObjectMeta{
   729  			Name:      projectName,
   730  			Namespace: bannerGUID,
   731  		},
   732  	}
   733  	s.Require().Eventually(func() bool {
   734  		err := s.Client.Get(s.ctx, types.NamespacedName{
   735  			Name:      project.Name,
   736  			Namespace: project.Namespace,
   737  		}, project)
   738  		return err == nil
   739  	}, s.timeout, s.tick, "expected Project was never found")
   740  	s.Require().True(isOwnedByBanner(project, bannerGUID), "expected (namespaced) Project to be owned by (cluster scoped) Banner")
   741  	// there is no Project controller to do this for us. simulate a ready resource
   742  	project.Status.Conditions = falsifyResourceReadiness(project.Status.Conditions)
   743  	project.Status.Number = &TestProjectNumber
   744  	s.Require().NoError(s.Client.Update(s.ctx, project))
   745  	s.Require().Eventually(func() bool {
   746  		err := s.Client.Get(s.ctx, types.NamespacedName{
   747  			Name:      banner.Name,
   748  			Namespace: banner.Namespace,
   749  		}, banner)
   750  		return err == nil && banner.Status.ProjectRef != ""
   751  	}, s.timeout, s.tick, "expected Banner Status was never set")
   752  
   753  	// Add 3 mock objects to Banner Inventory
   754  	banner.Status.Inventory = &inventory.ResourceInventory{}
   755  	banner.Status.Inventory.Entries = append(banner.Status.Inventory.Entries, generateMockObjects()...)
   756  	s.Require().NoError(s.Client.Update(s.ctx, banner))
   757  
   758  	// Verify GCP Project was created and matches Banner status
   759  	projectViaStatusRef := &resourceAPI.Project{}
   760  	s.Require().Eventually(func() bool {
   761  		err := s.Client.Get(s.ctx, types.NamespacedName{
   762  			Name:      strings.Split(banner.Status.ProjectRef, "/")[1],
   763  			Namespace: strings.Split(banner.Status.ProjectRef, "/")[0],
   764  		}, projectViaStatusRef)
   765  		return err == nil
   766  	}, s.timeout, s.tick, "expected Project was never found")
   767  	s.Equal(projectViaStatusRef, project, "could not find Project obj via Banner.Status.ProjectRef")
   768  
   769  	// K8s Namespace with correct project annotation
   770  	namespace := &corev1.Namespace{}
   771  	s.Require().Eventually(func() bool {
   772  		err := s.Client.Get(s.ctx, types.NamespacedName{
   773  			Name: bannerGUID,
   774  		}, namespace)
   775  		if err != nil {
   776  			return false
   777  		}
   778  		nsAnnos := namespace.Annotations
   779  		if nsAnnos != nil {
   780  			return s.Equal(generatedProjID, namespace.Annotations[meta.ProjectAnnotation])
   781  		}
   782  		return false
   783  	}, s.timeout, s.tick, "expected Namespace was never found or didnt have correct annotation")
   784  
   785  	//cluster := &edgeCluster.Cluster{}
   786  	//s.Require().Eventually(func() bool {
   787  	//	err := s.Client.Get(s.ctx, types.NamespacedName{
   788  	//		Name: bannerconstants.CreateBannerClusterInfraName(banner.Spec.DisplayName),
   789  	//	}, cluster)
   790  	//	return err == nil
   791  	//}, s.timeout, s.tick, "expected edge cluster was never found")
   792  	//s.Require().True(isOwnedByBanner(cluster, bannerGUID), "expected edge cluster to be owned by Banner")
   793  
   794  	// APIs (Service object)
   795  	s.Require().Eventually(func() bool {
   796  		for _, api := range gcpinfra.TenantAPIs {
   797  			service := &serviceAPI.Service{}
   798  			err := s.Client.Get(s.ctx, types.NamespacedName{
   799  				Name:      api,
   800  				Namespace: bannerGUID,
   801  			}, service)
   802  			if errors.IsNotFound(err) {
   803  				return false
   804  			}
   805  			s.Require().True(isOwnedByBanner(service, bannerGUID), "expected Service to be owned by Banner")
   806  			// if the service was created, force it to be ready, it wont ever be ready in the test env
   807  			// TODO: integration
   808  			service.Status.Conditions = falsifyResourceReadiness(service.Status.Conditions)
   809  			s.Require().NoError(s.Client.Update(s.ctx, service))
   810  		}
   811  		return true
   812  	}, s.timeout, s.tick, "not all Services were created (enabled)")
   813  
   814  	// StorageBucket and roles
   815  	sb := &storageAPI.StorageBucket{}
   816  	s.Require().Eventually(func() bool {
   817  		err := s.Client.Get(s.ctx, types.NamespacedName{
   818  			Name:      generatedProjID,
   819  			Namespace: bannerGUID,
   820  		}, sb)
   821  		return err == nil
   822  	}, s.timeout, s.tick, "expected StorageBucket was never found")
   823  	s.Require().True(isOwnedByBanner(sb, bannerGUID), "expected StorageBucket to be owned by Banner")
   824  
   825  	// Integration? platform secrets and metrics scope from real gcp
   826  	if integration.IsIntegrationTest() {
   827  		// TODO: Assert real values from integration test secretmanager. The expected value
   828  		// should come from test project values instead of hardcoded mocks
   829  		for _, secretID := range edgeconstants.PlatformSecretIDs {
   830  			v, err := s.sMgr.clients[bannerGUID].GetLatestSecretValue(s.ctx, secretID)
   831  			s.NoError(err)
   832  			s.Equal(("mocked secret value"), string(v))
   833  		}
   834  	} else {
   835  		// Platform secrets
   836  		s.Require().Eventually(func() bool {
   837  			c := s.sMgr.clients[generatedProjID]
   838  			if c == nil {
   839  				return false
   840  			}
   841  			for _, secretID := range edgeconstants.PlatformSecretIDs {
   842  				v, err := c.GetLatestSecretValue(s.ctx, secretID)
   843  				if err != nil {
   844  					return false
   845  				}
   846  				if !s.Equal(DefaultSecretValue, string(v)) {
   847  					return false
   848  				}
   849  			}
   850  			return true
   851  		}, s.timeout, s.tick, "never got a generated GCP Project ID")
   852  	}
   853  
   854  	s.Require().Eventually(func() bool {
   855  		err := s.Client.Get(s.ctx, types.NamespacedName{
   856  			Name:      banner.Name,
   857  			Namespace: banner.Namespace,
   858  		}, banner)
   859  		return err == nil && kmeta.IsStatusConditionTrue(banner.Status.Conditions, status.ReadyCondition)
   860  	}, s.timeout, s.tick, "Banner never became Ready")
   861  
   862  	// Checking GCP infra for KCC last since it depends on the cluster-infra cluster's EdgeID
   863  	// var hash string
   864  	// var kccResourceName string
   865  
   866  	hash := uuid.FromUUID(banner.Status.ClusterInfraClusterEdgeID).Hash()
   867  	kccResourceName := fmt.Sprintf("kcc-%s", hash)
   868  
   869  	infraIAMSA := &v1beta1.IAMServiceAccount{}
   870  	s.Require().Eventually(func() bool {
   871  		// if banner.Status.ClusterInfraClusterEdgeID != "" {
   872  		// 	hash = uuid.FromUUID(banner.Status.ClusterInfraClusterEdgeID).Hash()
   873  		// 	kccResourceName = fmt.Sprintf("kcc-%s", hash)
   874  		// }
   875  		err := s.Client.Get(s.ctx, types.NamespacedName{
   876  			Name:      kccResourceName,
   877  			Namespace: bannerGUID,
   878  		}, infraIAMSA)
   879  		return err == nil
   880  	}, s.timeout, s.tick, "expected kcc IAMServiceAccount was never found")
   881  	s.Require().True(isOwnedByBanner(infraIAMSA, bannerGUID), "expected IAMServiceAccount to be owned by Banner")
   882  	s.Require().Equal(fmt.Sprintf("k8s cfg connector for project %s", banner.Spec.DisplayName), *infraIAMSA.Spec.DisplayName)
   883  
   884  	projectOwner := &v1beta1.IAMPolicyMember{}
   885  	s.Require().Eventually(func() bool {
   886  		err := s.Client.Get(s.ctx, types.NamespacedName{
   887  			Name:      fmt.Sprintf("%s-owner", kccResourceName),
   888  			Namespace: bannerGUID,
   889  		}, projectOwner)
   890  		return err == nil
   891  	}, s.timeout, s.tick, "expected kcc project owner IAMPolicyMember was never found")
   892  	s.Require().True(isOwnedByBanner(projectOwner, bannerGUID), "expected IAMPolicyMember to be owned by Banner")
   893  	s.Require().Equal(fmt.Sprintf("serviceAccount:%s@%s.iam.gserviceaccount.com", kccResourceName, generatedProjID), *projectOwner.Spec.Member)
   894  	s.Require().Equal(resourceAPI.SchemeGroupVersion.String(), projectOwner.Spec.ResourceRef.APIVersion)
   895  	s.Require().Equal(resourceAPI.ProjectGVK.Kind, projectOwner.Spec.ResourceRef.Kind)
   896  	s.Require().Equal(generatedProjID, projectOwner.Spec.ResourceRef.External)
   897  	s.Require().Equal(roles.Owner, projectOwner.Spec.Role)
   898  
   899  	wiMember := &v1beta1.IAMPolicyMember{}
   900  	s.Require().Eventually(func() bool {
   901  		err := s.Client.Get(s.ctx, types.NamespacedName{
   902  			Name:      fmt.Sprintf("%s-workload-id", kccResourceName),
   903  			Namespace: bannerGUID,
   904  		}, wiMember)
   905  		return err == nil
   906  	}, s.timeout, s.tick, "expected kcc workload identity IAMPolicyMember was never found")
   907  	s.Require().True(isOwnedByBanner(wiMember, bannerGUID), "expected IAMPolicyMember to be owned by Banner")
   908  	s.Require().Equal(fmt.Sprintf("serviceAccount:%s.svc.id.goog[cnrm-system/cnrm-controller-manager]", generatedProjID), *wiMember.Spec.Member)
   909  	s.Require().Equal(v1beta1.SchemeGroupVersion.String(), wiMember.Spec.ResourceRef.APIVersion)
   910  	s.Require().Equal(v1beta1.IAMServiceAccountGVK.Kind, wiMember.Spec.ResourceRef.Kind)
   911  	s.Require().Equal(kccResourceName, wiMember.Spec.ResourceRef.Name)
   912  	s.Require().Equal(roles.WorkloadIdentityUser, wiMember.Spec.Role)
   913  
   914  	storageMember := &v1beta1.IAMPolicyMember{}
   915  	s.Require().Eventually(func() bool {
   916  		err := s.Client.Get(s.ctx, types.NamespacedName{
   917  			Name:      fmt.Sprintf("%s-storage-siemwriter", kccResourceName),
   918  			Namespace: bannerGUID,
   919  		}, storageMember)
   920  		return err == nil
   921  	}, s.timeout, s.tick, "expected kcc StorageBucket IAMPolicyMember was never found")
   922  	s.Require().True(isOwnedByBanner(storageMember, bannerGUID), "expected IAMPolicyMember to be owned by Banner")
   923  	s.Require().Equal(fmt.Sprintf("serviceAccount:service-%s@gcp-sa-logging.iam.gserviceaccount.com", TestProjectNumber), *storageMember.Spec.Member)
   924  	s.Require().Equal(storageAPI.SchemeGroupVersion.String(), storageMember.Spec.ResourceRef.APIVersion)
   925  	s.Require().Equal(storageAPI.StorageBucketGVK.Kind, storageMember.Spec.ResourceRef.Kind)
   926  	s.Require().Equal(fmt.Sprintf("%s-siem", s.projectID), storageMember.Spec.ResourceRef.External)
   927  	s.Require().Equal(roles.StorageObjectCreator, storageMember.Spec.Role)
   928  
   929  	// check inventory pruning
   930  	s.Eventually(func() bool {
   931  		err := s.Client.Get(s.ctx, types.NamespacedName{
   932  			Name:      banner.Name,
   933  			Namespace: banner.Namespace,
   934  		}, banner)
   935  		return err == nil && len(banner.Status.Inventory.Entries) == 36
   936  	}, s.timeout, s.tick, fmt.Sprintf("expected inventory not created %d", len(banner.Status.Inventory.Entries)))
   937  
   938  	s.checkInfraStatusDatabaseValue(banner, edgedb.InfraStatusReady)
   939  }
   940  
   941  // generateMockObjects generate 3 mock inventory objects
   942  // used to test that pruning works correctly.
   943  func generateMockObjects() []inventory.ResourceRef {
   944  	return []inventory.ResourceRef{
   945  		{
   946  			ID:      "abc-123-456",
   947  			Version: "v1",
   948  		},
   949  		{
   950  			ID:      "abc-456-789",
   951  			Version: "v1",
   952  		},
   953  		{
   954  			ID:      "abc-789-123",
   955  			Version: "v1",
   956  		},
   957  	}
   958  }
   959  
   960  func (s *Suite) TestBannerDeletion_ResourceLifecycle() {
   961  	integration.SkipIfNot(s.Framework)
   962  	// TODO. Deletion lifecycle (doesnt work in limited envtest setup, integration only)
   963  	bannerGUID := "guid-abc-123"
   964  	banner := &bannerAPI.Banner{
   965  		ObjectMeta: metav1.ObjectMeta{
   966  			Name: bannerGUID,
   967  		},
   968  		Spec: bannerAPI.BannerSpec{
   969  			DisplayName: "Testy McBanner",
   970  			BSL: bannerAPI.BSLConfig{
   971  				Organization: bannerAPI.BSLOrganization{
   972  					Name: "test-org",
   973  				},
   974  			},
   975  		},
   976  	}
   977  	project := &resourceAPI.Project{
   978  		ObjectMeta: metav1.ObjectMeta{
   979  			Name:      projectName,
   980  			Namespace: bannerGUID,
   981  		},
   982  	}
   983  	s.Require().Eventually(func() bool {
   984  		err := s.Client.Get(s.ctx, types.NamespacedName{
   985  			Name:      project.Name,
   986  			Namespace: project.Namespace,
   987  		}, project)
   988  		return err == nil
   989  	}, s.timeout, s.tick, "expected Project was never found")
   990  	s.Require().NoError(s.Client.Create(s.ctx, banner))
   991  	s.Require().NoError(s.Client.Delete(s.ctx, banner))
   992  	s.Require().Eventually(func() bool {
   993  		err := s.Client.Get(s.ctx, types.NamespacedName{
   994  			Name:      banner.Name,
   995  			Namespace: banner.Namespace,
   996  		}, banner)
   997  		return errors.IsNotFound(err)
   998  	}, s.timeout, s.tick, "test Banner was never deleted")
   999  	// Project owned by Banner should be garbage collected
  1000  	s.Require().Eventually(func() bool {
  1001  		p := &resourceAPI.Project{}
  1002  		err := s.Client.Get(s.ctx, types.NamespacedName{
  1003  			Name:      project.Name,
  1004  			Namespace: project.Namespace,
  1005  		}, p)
  1006  		return errors.IsNotFound(err)
  1007  	}, s.timeout, s.tick, "banner Project was never deleted")
  1008  
  1009  	// TODO: All children of project (gcp resources) should be cleaned up
  1010  }
  1011  
  1012  func (s *Suite) TestBannerCreation_CouchdbConfig() {
  1013  	bannerGUID := uuid.New().UUID
  1014  	generatedProjID := fmt.Sprintf("%s-%s", gcpinfra.ProjectIDPrefix, gcpProject.RandAN(29-(len(gcpinfra.ProjectIDPrefix))))
  1015  	banner := &bannerAPI.Banner{
  1016  		ObjectMeta: metav1.ObjectMeta{
  1017  			Name: bannerGUID,
  1018  		},
  1019  		Spec: bannerAPI.BannerSpec{
  1020  			DisplayName: "couchdb-enabled",
  1021  			GCP: bannerAPI.GCPConfig{
  1022  				ProjectID: generatedProjID,
  1023  			},
  1024  			Enablements: []string{
  1025  				"couchdb",
  1026  			},
  1027  			BSL: bannerAPI.BSLConfig{
  1028  				EnterpriseUnit: bannerAPI.BSLEnterpriseUnit{
  1029  					ID: uuid.New().UUID,
  1030  				},
  1031  				Organization: bannerAPI.BSLOrganization{
  1032  					Name: "test-org-dev-4",
  1033  				},
  1034  			},
  1035  		},
  1036  	}
  1037  	s.createBanner(banner)
  1038  	defer s.deleteBanner(banner)
  1039  
  1040  	// Secondary resources that should be created for a Couchdb Enabled Banner:
  1041  	// 1. GCP Project KCC resource is created and added to Banner status
  1042  	project := &resourceAPI.Project{
  1043  		ObjectMeta: metav1.ObjectMeta{
  1044  			Name:      projectName,
  1045  			Namespace: bannerGUID,
  1046  		},
  1047  	}
  1048  	s.Require().Eventually(func() bool {
  1049  		err := s.Client.Get(s.ctx, types.NamespacedName{
  1050  			Name:      project.Name,
  1051  			Namespace: project.Namespace,
  1052  		}, project)
  1053  		return err == nil
  1054  	}, s.timeout, s.tick, "expected Project was never found")
  1055  	s.Require().True(isOwnedByBanner(project, bannerGUID), "expected (namespaced) Project to be owned by (cluster scoped) Banner")
  1056  	project.Status.Conditions = falsifyResourceReadiness(project.Status.Conditions)
  1057  	project.Status.Number = &TestProjectNumber
  1058  	s.Require().NoError(s.Client.Update(s.ctx, project))
  1059  	s.Require().Eventually(func() bool {
  1060  		err := s.Client.Get(s.ctx, types.NamespacedName{
  1061  			Name:      banner.Name,
  1062  			Namespace: banner.Namespace,
  1063  		}, banner)
  1064  		return err == nil && banner.Status.ProjectRef != ""
  1065  	}, s.timeout, s.tick, "expected Banner Status was never set")
  1066  
  1067  	// 2. APIs (Service object)
  1068  	s.Require().Eventually(func() bool {
  1069  		for _, api := range gcpinfra.TenantAPIs {
  1070  			service := &serviceAPI.Service{}
  1071  			err := s.Client.Get(s.ctx, types.NamespacedName{
  1072  				Name:      api,
  1073  				Namespace: bannerGUID,
  1074  			}, service)
  1075  			if errors.IsNotFound(err) {
  1076  				return false
  1077  			}
  1078  			s.Require().True(isOwnedByBanner(service, bannerGUID), "expected Service to be owned by Banner")
  1079  			s.Require().Equal(meta.DeletionPolicyAbandon, service.Annotations[meta.DeletionPolicyAnnotation])
  1080  			// if the service was created, force it to be ready, it wont ever be ready in the test env
  1081  			// TODO: integration
  1082  			service.Status.Conditions = falsifyResourceReadiness(service.Status.Conditions)
  1083  			s.Require().NoError(s.Client.Update(s.ctx, service))
  1084  		}
  1085  		return true
  1086  	}, s.timeout, s.tick, "not all Services were created (enabled)")
  1087  
  1088  	// Banner eventually becomes ready
  1089  	s.Require().Eventually(func() bool {
  1090  		err := s.Client.Get(s.ctx, types.NamespacedName{
  1091  			Name:      banner.Name,
  1092  			Namespace: banner.Namespace,
  1093  		}, banner)
  1094  		return err == nil && kmeta.IsStatusConditionTrue(banner.Status.Conditions, status.ReadyCondition)
  1095  	}, s.timeout, s.tick, "Banner never became Ready")
  1096  
  1097  	s.checkInfraStatusDatabaseValue(banner, edgedb.InfraStatusReady)
  1098  }
  1099  
  1100  func (s *Suite) TestBannerControllerSetsInfraStatusError() {
  1101  	bannerGUID := uuid.New().UUID
  1102  	testBSLID := uuid.New().UUID
  1103  	testOrg := uuid.New().UUID
  1104  	// Set an invalid project id for the banner to get an error status
  1105  	banner := &bannerAPI.Banner{
  1106  		ObjectMeta: metav1.ObjectMeta{
  1107  			Name: bannerGUID,
  1108  		},
  1109  		Spec: bannerAPI.BannerSpec{
  1110  			DisplayName: "test-infra-status-error",
  1111  			GCP: bannerAPI.GCPConfig{
  1112  				ProjectID: "invalid projectID",
  1113  			},
  1114  			BSL: bannerAPI.BSLConfig{
  1115  				EnterpriseUnit: bannerAPI.BSLEnterpriseUnit{
  1116  					ID: testBSLID,
  1117  				},
  1118  				Organization: bannerAPI.BSLOrganization{
  1119  					Name: testOrg,
  1120  				},
  1121  			},
  1122  		},
  1123  	}
  1124  
  1125  	s.createBanner(banner)
  1126  	defer s.deleteBanner(banner)
  1127  
  1128  	// Ensure the banner reconciler properly set the infra_status to ERROR
  1129  	s.checkInfraStatusDatabaseValue(banner, edgedb.InfraStatusError)
  1130  }
  1131  
  1132  func (s *Suite) createBanner(banner *bannerAPI.Banner) {
  1133  	bannerType := string(bannerconstants.EU)
  1134  	tenantID := uuid.New().UUID
  1135  	s.Require().NoError(banner.IsValid())
  1136  	_, err := s.db.Exec("INSERT INTO tenants (tenant_edge_id, org_id, org_name) VALUES ($1, $2, $3);", tenantID, banner.Spec.BSL.EnterpriseUnit.ID, banner.Spec.BSL.Organization.Name)
  1137  	s.NoError(err)
  1138  	const stmt = "INSERT INTO banners (banner_bsl_id, banner_name, banner_type, project_id, tenant_edge_id, banner_edge_id, description) VALUES ($1, $2, $3, $4, $5, $6, $7)"
  1139  	_, err = s.db.Exec(stmt, banner.Spec.BSL.EnterpriseUnit.ID, banner.Spec.DisplayName, bannerType, banner.Spec.GCP.ProjectID, tenantID, banner.Name, nil)
  1140  	s.Require().NoError(err)
  1141  	s.Require().NoError(s.Client.Create(s.ctx, banner))
  1142  }
  1143  
  1144  func (s *Suite) checkInfraStatusDatabaseValue(banner *bannerAPI.Banner, infraStatus edgedb.InfraStatus) {
  1145  	const stmt = "SELECT infra_status FROM banners WHERE banner_edge_id=$1"
  1146  	s.Require().Eventually(func() bool {
  1147  		var v string
  1148  		err := s.db.QueryRow(stmt, banner.Name).Scan(&v)
  1149  		if err != nil {
  1150  			return false
  1151  		}
  1152  		return v == string(infraStatus)
  1153  	}, s.timeout, s.tick, "expected infra_status value %q not set in database", infraStatus)
  1154  }
  1155  
  1156  func (s *Suite) validateEdgeLabels(banner *bannerAPI.Banner) {
  1157  	const stmt = "SELECT label_key, visible, editable, color, description, label_type FROM labels WHERE banner_edge_id=$1 AND label_type=$2 AND label_key=$3"
  1158  
  1159  	expectedEdgeLabels := capabilities.EdgeAutomatedCapabilityLabels
  1160  	expectedEdgeLabels = append(expectedEdgeLabels, fetchEdgeCapabilityLabels()...)
  1161  	for _, label := range expectedEdgeLabels {
  1162  		var actualLabel = &model.LabelInput{}
  1163  		s.Require().Eventually(func() bool {
  1164  			row := s.db.QueryRowContext(s.ctx, stmt, banner.Name, label.Type, label.Key)
  1165  			if err := row.Scan(&actualLabel.Key, &actualLabel.Visible, &actualLabel.Editable, &actualLabel.Color, &actualLabel.Description, &actualLabel.Type); err != nil {
  1166  				return false
  1167  			}
  1168  
  1169  			actualLabel.BannerEdgeID = banner.GetName()
  1170  			label.BannerEdgeID = banner.GetName()
  1171  			return cmp.Equal(label, actualLabel)
  1172  		}, s.timeout, s.tick, "expected label", label, "actual label", actualLabel)
  1173  	}
  1174  }
  1175  
  1176  func (s *Suite) deleteBanner(banner *bannerAPI.Banner) {
  1177  	s.NoError(s.Client.Delete(s.ctx, banner))
  1178  	s.Eventually(func() bool {
  1179  		return errors.IsNotFound(s.Client.Get(s.ctx, client.ObjectKeyFromObject(banner), banner))
  1180  	}, s.timeout, s.tick, "banner resource was never deleted %v", banner)
  1181  }
  1182  

View as plain text