...

Source file src/github.com/GoogleCloudPlatform/k8s-config-connector/operator/tests/e2e/e2e_test.go

Documentation: github.com/GoogleCloudPlatform/k8s-config-connector/operator/tests/e2e

     1  // Copyright 2022 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package e2e
    16  
    17  import (
    18  	"bytes"
    19  	"context"
    20  	"flag"
    21  	"fmt"
    22  	"io/ioutil"
    23  	"os"
    24  	"os/exec"
    25  	"path"
    26  	"strings"
    27  	"testing"
    28  	"time"
    29  
    30  	"github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/k8s"
    31  	"github.com/GoogleCloudPlatform/k8s-config-connector/operator/scripts/utils"
    32  	kcck8s "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/k8s"
    33  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/randomid"
    34  	testgcp "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/gcp"
    35  
    36  	"github.com/blang/semver"
    37  	"github.com/cenkalti/backoff"
    38  	"github.com/go-logr/logr"
    39  	"github.com/go-logr/zapr"
    40  	"go.uber.org/zap"
    41  	"google.golang.org/api/cloudbilling/v1"
    42  	"google.golang.org/api/cloudresourcemanager/v1"
    43  	containerBeta "google.golang.org/api/container/v1beta1"
    44  	"google.golang.org/api/iam/v1"
    45  	v1 "k8s.io/api/core/v1"
    46  	"k8s.io/apimachinery/pkg/api/errors"
    47  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    48  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    49  	"k8s.io/apimachinery/pkg/util/wait"
    50  	"k8s.io/client-go/kubernetes"
    51  	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp" // Load the gcp auth plugin
    52  	"k8s.io/client-go/tools/clientcmd"
    53  )
    54  
    55  const (
    56  	SERVICE_ACC_ID = "cnrm-system"
    57  	SECRET_NAME    = "gsa-key"
    58  
    59  	// projects, by default, are limited to 3 GKE clusters per region, for that reason we run the tests in two regions
    60  	GKE_CLUSTER_ZONE1      = "us-central1-a"
    61  	GKE_CLUSTER_ZONE2      = "us-west1-a"
    62  	GKE_CLUSTER_ZONE3      = "us-west2-a"
    63  	KUBECTL_DELETE_TIMEOUT = 5 * time.Minute
    64  
    65  	OPERATOR_RELEASE_BUCKET  = "kcc-operator-internal"
    66  	OPERATOR_RELEASE_TARBALL = "release-bundle.tar.gz"
    67  	KCC_RELEASE_BUCKET       = "cnrm"
    68  	KCC_RELEASE_TARBALL      = "release-bundle.tar.gz"
    69  	// Use ConfigConnector operator 1.95.0 as the base version for upgrade test
    70  	// because previous versions cannot be installed onto K8s 1.26+ (default GKE
    71  	// cluster version as of 6/27/2023) due to the removal of
    72  	// autoscaling/v2beta2.
    73  	BASE_VERSION_SHA = "4119846"
    74  )
    75  
    76  var (
    77  	SERVICES = []string{
    78  		"container.googleapis.com",
    79  		"iamcredentials.googleapis.com",
    80  		"artifactregistry.googleapis.com",
    81  	}
    82  	organization        = testgcp.GetOrgID(nil)
    83  	billingAccount      = testgcp.GetBillingAccountID(nil)
    84  	f                   = &flags{}
    85  	defaultBackOff      = wait.Backoff{Steps: 5, Duration: 500 * time.Millisecond, Factor: 1.5}
    86  	longIntervalBackOff = wait.Backoff{Steps: 3, Duration: 2 * time.Minute, Factor: 1}
    87  )
    88  
    89  type TestOptions struct {
    90  	OrganizationID     string
    91  	BillingAccountID   string
    92  	ServiceAccountID   string
    93  	GKEClusterLocation string
    94  	BaseVersionSHA     string
    95  	ProjectID          string
    96  	SecretName         string
    97  }
    98  
    99  type flags struct {
   100  	projectID string
   101  	cleanup   bool
   102  	version   string
   103  }
   104  
   105  type cluster struct {
   106  	kubectl   *kubectl
   107  	clientset *kubernetes.Clientset
   108  	log       logr.Logger
   109  }
   110  
   111  type kubectl struct {
   112  	kubeconfigPath string
   113  	deleteTimeout  time.Duration
   114  }
   115  
   116  type configConnectorSample struct {
   117  	configConnectorClusterModeWorkloadIdentityYAMLPath string
   118  	configConnectorClusterModeGCPIdentityYAMLPath      string
   119  	configConnectorNamespacedModeYAMLPath              string
   120  	configConnectorContextYAMLPath                     string
   121  }
   122  
   123  type cleanupFunc func() error
   124  
   125  func TestMain(m *testing.M) {
   126  	flag.StringVar(&f.projectID, "project-id", "", "Project ID that will be used for the project created for E2E tests.")
   127  	flag.BoolVar(&f.cleanup, "cleanup", true, "If true, project and clusters created for testing will be deleted before exiting the test suite. "+
   128  		"Set to false if you want to keep clusters for debugging when running the test locally.")
   129  	flag.StringVar(&f.version, "version", "latest", "Version of the KCC Operator to use for E2E tests. "+
   130  		"The version of the KCC Operator is defined by SHORT_SHA in release.sh. Use the corresponding SHORT_SHA value to run the e2e test against a particular commit. "+
   131  		"The default value is 'latest', which represents the version of the last green canary candidate promoted by the periodic-kcc-operator-release prow job.")
   132  	flag.Parse()
   133  	if f.projectID == "" {
   134  		fmt.Println("error parsing command line flags: project-id is required")
   135  		os.Exit(1)
   136  	}
   137  
   138  	log, err := newLogger("TestMain")
   139  	if err != nil {
   140  		fmt.Printf("error creating logger: %v", err)
   141  		os.Exit(1)
   142  	}
   143  
   144  	log.Info("Setting up a project for E2E tests...")
   145  	deleteProject, err := setupProject(organization, f.projectID, billingAccount, SERVICE_ACC_ID, log)
   146  	if err != nil {
   147  		log.Error(err, "error setting up project\r\n",
   148  			"Organization", organization, "ProjectID", f.projectID,
   149  			"BillingAccount", billingAccount, "ServiceID", SERVICE_ACC_ID)
   150  		cleanUpProject(deleteProject, f.cleanup, log)
   151  		os.Exit(1)
   152  	}
   153  	log.Info("Beginning tests...")
   154  	exitCode := m.Run()
   155  	cleanUpProject(deleteProject, f.cleanup, log)
   156  	os.Exit(exitCode)
   157  }
   158  
   159  func TestKCCInstallAndUninstall_Namespaced(t *testing.T) {
   160  	t.Parallel()
   161  	testOptions := newTestOptions()
   162  	testOptions.GKEClusterLocation = GKE_CLUSTER_ZONE1
   163  	testId, log, cluster, teardown := setup(t, testOptions)
   164  	if f.cleanup {
   165  		defer teardown()
   166  	}
   167  
   168  	manifestsDir, sample, err := getOperatorReleaseAssetsForVersion(f.version, testOptions.ServiceAccountID, testOptions.ProjectID, log)
   169  	if err != nil {
   170  		t.Fatalf("error getting operator release assets for version '%v': %v", f.version, err)
   171  	}
   172  	log.Info("Installing the operator...")
   173  	if err := cluster.installOperator(manifestsDir); err != nil {
   174  		t.Fatalf("error installing the operator: %v", err)
   175  	}
   176  	log.Info("Installing KCC...")
   177  	if err := cluster.installKCC(sample.configConnectorNamespacedModeYAMLPath); err != nil {
   178  		t.Fatalf("error installing KCC: %v", err)
   179  	}
   180  	namespace := "e2e-test-namespace"
   181  	if err := cluster.createNamespace(namespace); err != nil {
   182  		t.Fatalf("error creating namespace '%v': %v", namespace, err)
   183  	}
   184  	if err := cluster.enableKCCForNamespace(namespace, sample.configConnectorContextYAMLPath, testOptions.ServiceAccountID, testOptions.ProjectID); err != nil {
   185  		t.Fatalf("error enabling KCC for namespace '%v': %v", namespace, err)
   186  	}
   187  	if err := cluster.addProjectIDAnnotationToNamespace(namespace, f.projectID); err != nil {
   188  		t.Fatalf("error annotating namespace '%v' with the project ID: %v", namespace, err)
   189  	}
   190  	kccVersion, err := cluster.getKCCVersion()
   191  	if err != nil {
   192  		t.Fatalf("error determining KCC version: %v", err)
   193  	}
   194  	log.Info("Downloading and extracting KCC release tarball ...", "version", kccVersion)
   195  	kccReleaseAssetsDir, err := createTempDir("e2e-kcc-release-assets")
   196  	if err != nil {
   197  		t.Fatalf("error creating temporary directory for KCC release assets: %v", err)
   198  	}
   199  	if err := downloadAndExtractKCCReleaseTarball(kccVersion, kccReleaseAssetsDir); err != nil {
   200  		t.Fatalf("error downloading and extracting KCC with version '%v': %v", kccVersion, err)
   201  	}
   202  	repoName, repoYAMLDir, err := getArtifactRegistryRepositorySample(kccReleaseAssetsDir, testId)
   203  	if err != nil {
   204  		t.Fatalf("error getting ArtifactRegistryRepository sample from KCC release assets: %v", err)
   205  	}
   206  	log.Info("Creating ArtifactRegistryRepository...")
   207  	if err := cluster.createArtifactRegistryRepository(namespace, repoName, repoYAMLDir); err != nil {
   208  		t.Fatalf("error creating ArtifactRegistryRepository: %v", err)
   209  	}
   210  	log.Info("Deleting ArtifactRegistryRepository...")
   211  	if err := cluster.deleteArtifactRegistryRepository(namespace, repoName); err != nil {
   212  		t.Fatal(err)
   213  	}
   214  	log.Info("Uninstalling KCC...")
   215  	if err := cluster.uninstallKCC(); err != nil {
   216  		t.Fatalf("error uninstalling KCC: %v", err)
   217  	}
   218  }
   219  
   220  func TestKCCInstallAnd_Delete_Namespace_In_Namespaced_Mode(t *testing.T) {
   221  	t.Parallel()
   222  	testOptions := newTestOptions()
   223  	testOptions.GKEClusterLocation = GKE_CLUSTER_ZONE3
   224  	testId, log, cluster, teardown := setup(t, testOptions)
   225  	if f.cleanup {
   226  		defer teardown()
   227  	}
   228  	manifestsDir, sample, err := getOperatorReleaseAssetsForVersion(f.version, testOptions.ServiceAccountID, testOptions.ProjectID, log)
   229  	if err != nil {
   230  		t.Fatalf("error getting operator release assets for version '%v': %v", f.version, err)
   231  	}
   232  	log.Info("Installing the operator...")
   233  	if err := cluster.installOperator(manifestsDir); err != nil {
   234  		t.Fatalf("error installing the operator: %v", err)
   235  	}
   236  	log.Info("Installing KCC...")
   237  	if err := cluster.installKCC(sample.configConnectorNamespacedModeYAMLPath); err != nil {
   238  		t.Fatalf("error installing KCC: %v", err)
   239  	}
   240  	namespace := "e2e-test-namespace"
   241  	if err := cluster.createNamespace(namespace); err != nil {
   242  		t.Fatalf("error creating namespace '%v': %v", namespace, err)
   243  	}
   244  	if err := cluster.enableKCCForNamespace(namespace, sample.configConnectorContextYAMLPath, testOptions.ServiceAccountID, testOptions.ProjectID); err != nil {
   245  		t.Fatalf("error enabling KCC for namespace '%v': %v", namespace, err)
   246  	}
   247  	if err := cluster.addProjectIDAnnotationToNamespace(namespace, f.projectID); err != nil {
   248  		t.Fatalf("error annotating namespace '%v' with the project ID: %v", namespace, err)
   249  	}
   250  	kccVersion, err := cluster.getKCCVersion()
   251  	if err != nil {
   252  		t.Fatalf("error determining KCC version: %v", err)
   253  	}
   254  	log.Info("Downloading and extracting KCC release tarball ...", "version", kccVersion)
   255  	kccReleaseAssetsDir, err := createTempDir("e2e-kcc-release-assets")
   256  	if err != nil {
   257  		t.Fatalf("error creating temporary directory for KCC release assets: %v", err)
   258  	}
   259  	if err := downloadAndExtractKCCReleaseTarball(kccVersion, kccReleaseAssetsDir); err != nil {
   260  		t.Fatalf("error downloading and extracting KCC with version '%v': %v", kccVersion, err)
   261  	}
   262  	repoName, repoYAMLDir, err := getArtifactRegistryRepositorySample(kccReleaseAssetsDir, testId)
   263  	if err != nil {
   264  		t.Fatalf("error getting ArtifactRegistryRepository sample from KCC release assets: %v", err)
   265  	}
   266  	log.Info("Creating ArtifactRegistryRepository...")
   267  	if err := cluster.createArtifactRegistryRepository(namespace, repoName, repoYAMLDir); err != nil {
   268  		t.Fatalf("error creating ArtifactRegistryRepository: %v", err)
   269  	}
   270  	// add an extra finalizer to ensure resources are not deleted, the config-connector-operator should wait
   271  	// until all KCC resources are deleted before deleting the related KCC pods
   272  	log.Info("Adding custom finalizer to prevent deletion...")
   273  	extraFinalizer := "extra-finalizer"
   274  	if err := cluster.addFinalizerToArtifactRegistryRepository(namespace, repoName, extraFinalizer); err != nil {
   275  		t.Fatalf("error adding finalizer to ArtifactRegistryRepository: %v", err)
   276  	}
   277  	log.Info("Deleting Namespace...")
   278  	if err := cluster.deleteNamespace(namespace); err != nil {
   279  		t.Fatalf("error deleting namespace: %v", err)
   280  	}
   281  	// Sometimes, it takes a long time for k8s to cascade delete KCC resource CRs under the deleted namespace.
   282  	// Therefore we perform a direct deletion on the ArtifactRegistryRepository object to speed things up.
   283  	log.Info("Deleting ArtifactRegistryRepository...")
   284  	if err := cluster.deleteArtifactRegistryRepository(namespace, repoName, "--wait=false"); err != nil {
   285  		t.Fatal(err)
   286  	}
   287  	// The CNRM manager pod should still be running as the operator should wait until all CNRM resources deleted before
   288  	// deleting the manager pods. This check ensures the manager is able to remove its finalizer from the ArtifactRegistryRepository.
   289  	log.Info("Waiting for CNRM finalizer to be removed from ArtifactRegistryRepository...")
   290  	if err := cluster.waitForCNRMFinalizersToBeRemovedFromArtifactRegistryRepository(namespace, repoName); err != nil {
   291  		t.Fatalf("error waiting for CNRM finalizer to be removed from ArtifactRegistryRepository: %v", err)
   292  	}
   293  	// The config connector context should NOT be removed as the ArtifactRegistryRepository has not yet been removed due to its extra finalizer
   294  	log.Info("Verifying the ConfigConnectorContext still exists but is unhealthy...")
   295  	if err := cluster.waitForConfigConnectorContextToBeUnhealthy(namespace, k8s.ConfigConnectorContextAllowedName); err != nil {
   296  		t.Fatalf("error verifying the ConfigConnectorContext's health: %v", err)
   297  	}
   298  	log.Info("Removing custom finalizer to enable deletion...")
   299  	if err := cluster.removeFinalizerToArtifactRegistryRepository(namespace, repoName, extraFinalizer); err != nil {
   300  		t.Fatalf("error removing finalizer from ArtifactRegistryRepository: %v", err)
   301  	}
   302  	log.Info("Waiting for ConfigConnectorContext to be removed...")
   303  	if err := cluster.waitForConfigConnectorContextToBeRemoved(namespace, k8s.ConfigConnectorContextAllowedName); err != nil {
   304  		t.Fatalf("error waiting for ConfigConnectorContextToBeRemoved: %v", err)
   305  	}
   306  	log.Info("Waiting for namespace to be deleted...")
   307  	if err := cluster.waitForNamespaceToBeDeleted(namespace); err != nil {
   308  		t.Fatalf("error waiting for namespace to be deleted")
   309  	}
   310  }
   311  
   312  func TestKCCInstallAndUninstall_Cluster_WorkloadIdentity(t *testing.T) {
   313  	t.Parallel()
   314  	testOptions := newTestOptions()
   315  	testOptions.GKEClusterLocation = GKE_CLUSTER_ZONE2
   316  	testId, log, cluster, teardown := setup(t, testOptions)
   317  	if f.cleanup {
   318  		defer teardown()
   319  	}
   320  
   321  	manifestsDir, sample, err := getOperatorReleaseAssetsForVersion(f.version, testOptions.ServiceAccountID, testOptions.ProjectID, log)
   322  	if err != nil {
   323  		t.Fatalf("error getting operator release assets for version '%v': %v", f.version, err)
   324  	}
   325  	log.Info("Installing the operator...")
   326  	if err := cluster.installOperator(manifestsDir); err != nil {
   327  		t.Fatalf("error installing the operator: %v", err)
   328  	}
   329  	log.Info("Installing KCC in cluster mode with workload identity...")
   330  	if err := cluster.installKCC(sample.configConnectorClusterModeWorkloadIdentityYAMLPath); err != nil {
   331  		t.Fatalf("error installing KCC: %v", err)
   332  	}
   333  	namespace := "e2e-test-namespace"
   334  	if err := cluster.createNamespace(namespace); err != nil {
   335  		t.Fatalf("error creating namespace '%v': %v", namespace, err)
   336  	}
   337  	if err := cluster.addProjectIDAnnotationToNamespace(namespace, f.projectID); err != nil {
   338  		t.Fatalf("error annotating namespace '%v' with the project ID: %v", namespace, err)
   339  	}
   340  	kccVersion, err := cluster.getKCCVersion()
   341  	if err != nil {
   342  		t.Fatalf("error determining KCC version: %v", err)
   343  	}
   344  	log.Info("Downloading and extracting KCC release tarball ...", "version", kccVersion)
   345  	kccReleaseAssetsDir, err := createTempDir("e2e-kcc-release-assets")
   346  	if err != nil {
   347  		t.Fatalf("error creating temporary directory for KCC release assets: %v", err)
   348  	}
   349  	if err := downloadAndExtractKCCReleaseTarball(kccVersion, kccReleaseAssetsDir); err != nil {
   350  		t.Fatalf("error downloading and extracting KCC with version '%v': %v", kccVersion, err)
   351  	}
   352  	repoName, repoYAMLDir, err := getArtifactRegistryRepositorySample(kccReleaseAssetsDir, testId)
   353  	if err != nil {
   354  		t.Fatalf("error getting ArtifactRegistryRepository sample from KCC release assets: %v", err)
   355  	}
   356  	log.Info("Creating ArtifactRegistryRepository...")
   357  	if err := cluster.createArtifactRegistryRepository(namespace, repoName, repoYAMLDir); err != nil {
   358  		t.Fatalf("error creating ArtifactRegistryRepository: %v", err)
   359  	}
   360  	log.Info("Deleting ArtifactRegistryRepository...")
   361  	if err := cluster.deleteArtifactRegistryRepository(namespace, repoName); err != nil {
   362  		t.Fatal(err)
   363  	}
   364  	log.Info("Uninstalling KCC...")
   365  	if err := cluster.uninstallKCC(); err != nil {
   366  		t.Fatalf("error uninstalling KCC: %v", err)
   367  	}
   368  }
   369  
   370  func TestKCCInstallAndUninstall_Cluster_GCPIdentity(t *testing.T) {
   371  	t.Parallel()
   372  	testOptions := newTestOptions()
   373  	testOptions.GKEClusterLocation = GKE_CLUSTER_ZONE2
   374  	testOptions.SecretName = SECRET_NAME
   375  	testId, log, cluster, teardown := setup(t, testOptions)
   376  	if f.cleanup {
   377  		defer teardown()
   378  	}
   379  
   380  	manifestsDir, sample, err := getOperatorReleaseAssetsForVersion(f.version, testOptions.ServiceAccountID, testOptions.ProjectID, log)
   381  	if err != nil {
   382  		t.Fatalf("error getting operator release assets for version '%v': %v", f.version, err)
   383  	}
   384  	log.Info("Installing the operator...")
   385  	if err := cluster.installOperator(manifestsDir); err != nil {
   386  		t.Fatalf("error installing the operator: %v", err)
   387  	}
   388  	log.Info("Installing KCC in cluster mode with GCP identity...")
   389  	if err := cluster.installKCC(sample.configConnectorClusterModeGCPIdentityYAMLPath); err != nil {
   390  		t.Fatalf("error installing KCC: %v", err)
   391  	}
   392  	namespace := "e2e-test-namespace"
   393  	if err := cluster.createNamespace(namespace); err != nil {
   394  		t.Fatalf("error creating namespace '%v': %v", namespace, err)
   395  	}
   396  	if err := cluster.addProjectIDAnnotationToNamespace(namespace, f.projectID); err != nil {
   397  		t.Fatalf("error annotating namespace '%v' with the project ID: %v", namespace, err)
   398  	}
   399  	kccVersion, err := cluster.getKCCVersion()
   400  	if err != nil {
   401  		t.Fatalf("error determining KCC version: %v", err)
   402  	}
   403  	log.Info("Downloading and extracting KCC release tarball ...", "version", kccVersion)
   404  	kccReleaseAssetsDir, err := createTempDir("e2e-kcc-release-assets")
   405  	if err != nil {
   406  		t.Fatalf("error creating temporary directory for KCC release assets: %v", err)
   407  	}
   408  	if err := downloadAndExtractKCCReleaseTarball(kccVersion, kccReleaseAssetsDir); err != nil {
   409  		t.Fatalf("error downloading and extracting KCC with version '%v': %v", kccVersion, err)
   410  	}
   411  	repoName, repoYAMLDir, err := getArtifactRegistryRepositorySample(kccReleaseAssetsDir, testId)
   412  	if err != nil {
   413  		t.Fatalf("error getting ArtifactRegistryRepository sample from KCC release assets: %v", err)
   414  	}
   415  	log.Info("Creating ArtifactRegistryRepository...")
   416  	if err := cluster.createArtifactRegistryRepository(namespace, repoName, repoYAMLDir); err != nil {
   417  		t.Fatalf("error creating ArtifactRegistryRepository: %v", err)
   418  	}
   419  	log.Info("Deleting ArtifactRegistryRepository...")
   420  	if err := cluster.deleteArtifactRegistryRepository(namespace, repoName); err != nil {
   421  		t.Fatal(err)
   422  	}
   423  	log.Info("Uninstalling KCC...")
   424  	if err := cluster.uninstallKCC(); err != nil {
   425  		t.Fatalf("error uninstalling KCC: %v", err)
   426  	}
   427  }
   428  
   429  func TestKCCInstallAndUninstallWithoutDeletingKCCResources(t *testing.T) {
   430  	t.Parallel()
   431  	testOptions := newTestOptions()
   432  	testOptions.GKEClusterLocation = GKE_CLUSTER_ZONE1
   433  	testId, log, cluster, teardown := setup(t, testOptions)
   434  	if f.cleanup {
   435  		defer teardown()
   436  	}
   437  
   438  	manifestsDir, sample, err := getOperatorReleaseAssetsForVersion(f.version, testOptions.ServiceAccountID, testOptions.ProjectID, log)
   439  	if err != nil {
   440  		t.Fatalf("error getting operator release assets for version '%v': %v", f.version, err)
   441  	}
   442  	log.Info("Installing the operator...")
   443  	if err := cluster.installOperator(manifestsDir); err != nil {
   444  		t.Fatalf("error installing the operator: %v", err)
   445  	}
   446  	log.Info("Installing KCC...")
   447  	if err := cluster.installKCC(sample.configConnectorNamespacedModeYAMLPath); err != nil {
   448  		t.Fatalf("error installing KCC: %v", err)
   449  	}
   450  	namespace := "e2e-test-namespace"
   451  	if err := cluster.createNamespace(namespace); err != nil {
   452  		t.Fatalf("error creating namespace '%v': %v", namespace, err)
   453  	}
   454  	if err := cluster.enableKCCForNamespace(namespace, sample.configConnectorContextYAMLPath, testOptions.ServiceAccountID, testOptions.ProjectID); err != nil {
   455  		t.Fatalf("error enabling KCC for namespace '%v': %v", namespace, err)
   456  	}
   457  	if err := cluster.addProjectIDAnnotationToNamespace(namespace, f.projectID); err != nil {
   458  		t.Fatalf("error annotating namespace '%v' with the project ID: %v", namespace, err)
   459  	}
   460  	kccVersion, err := cluster.getKCCVersion()
   461  	if err != nil {
   462  		t.Fatalf("error determining KCC version: %v", err)
   463  	}
   464  	log.Info("Downloading and extracting KCC release tarball...", "version", kccVersion)
   465  	kccReleaseAssetsDir, err := createTempDir("e2e-kcc-release-assets")
   466  	if err != nil {
   467  		t.Fatalf("error creating temporary directory for KCC release assets: %v", err)
   468  	}
   469  	if err := downloadAndExtractKCCReleaseTarball(kccVersion, kccReleaseAssetsDir); err != nil {
   470  		t.Fatalf("error downloading and extracting KCC with version '%v': %v", kccVersion, err)
   471  	}
   472  	repoName, repoYAMLDir, err := getArtifactRegistryRepositorySample(kccReleaseAssetsDir, testId)
   473  	if err != nil {
   474  		t.Fatalf("error getting ArtifactRegistryRepository sample from KCC release assets: %v", err)
   475  	}
   476  	log.Info("Creating ArtifactRegistryRepository...")
   477  	if err := cluster.createArtifactRegistryRepository(namespace, repoName, repoYAMLDir); err != nil {
   478  		t.Fatalf("error creating ArtifactRegistryRepository: %v", err)
   479  	}
   480  	log.Info("Uninstalling KCC...")
   481  	if err := cluster.uninstallKCC(); err != nil {
   482  		t.Fatalf("error uninstalling KCC: %v", err)
   483  	}
   484  	if err := checkArtifactRegistryRepositoryExistsOnGCP(repoName, f.projectID); err != nil {
   485  		t.Fatal(err)
   486  	}
   487  }
   488  
   489  func TestShouldNotBeAbleToCreateKCCResourcesIfKCCNotEnabledForNamespace(t *testing.T) {
   490  	t.Parallel()
   491  	testOptions := newTestOptions()
   492  	testOptions.GKEClusterLocation = GKE_CLUSTER_ZONE1
   493  	testId, log, cluster, teardown := setup(t, testOptions)
   494  	if f.cleanup {
   495  		defer teardown()
   496  	}
   497  
   498  	manifestsDir, sample, err := getOperatorReleaseAssetsForVersion(f.version, testOptions.ServiceAccountID, testOptions.ProjectID, log)
   499  	if err != nil {
   500  		t.Fatalf("error getting operator release assets for version '%v': %v", f.version, err)
   501  	}
   502  	log.Info("Installing the operator...")
   503  	if err := cluster.installOperator(manifestsDir); err != nil {
   504  		t.Fatalf("error installing the operator: %v", err)
   505  	}
   506  	log.Info("Installing KCC...")
   507  	if err := cluster.installKCC(sample.configConnectorNamespacedModeYAMLPath); err != nil {
   508  		t.Fatalf("error installing KCC: %v", err)
   509  	}
   510  	namespace := "e2e-test-namespace"
   511  	if err := cluster.createNamespace(namespace); err != nil {
   512  		t.Fatalf("error creating namespace '%v': %v", namespace, err)
   513  	}
   514  	if err := cluster.addProjectIDAnnotationToNamespace(namespace, f.projectID); err != nil {
   515  		t.Fatalf("error annotating namespace '%v' with the project ID: %v", namespace, err)
   516  	}
   517  	kccVersion, err := cluster.getKCCVersion()
   518  	if err != nil {
   519  		t.Fatalf("error determining KCC version: %v", err)
   520  	}
   521  	log.Info("Downloading and extracting KCC release tarball ...", "version", kccVersion)
   522  	kccReleaseAssetsDir, err := createTempDir("e2e-kcc-release-assets")
   523  	if err != nil {
   524  		t.Fatalf("error creating temporary directory for KCC release assets: %v", err)
   525  	}
   526  	if err := downloadAndExtractKCCReleaseTarball(kccVersion, kccReleaseAssetsDir); err != nil {
   527  		t.Fatalf("error downloading and extracting KCC with version '%v': %v", kccVersion, err)
   528  	}
   529  	repoName, repoYAMLDir, err := getArtifactRegistryRepositorySample(kccReleaseAssetsDir, testId)
   530  	if err != nil {
   531  		t.Fatalf("error getting ArtifactRegistryRepository sample from KCC release assets: %v", err)
   532  	}
   533  	log.Info("Creating ArtifactRegistryRepository...")
   534  	if err := cluster.createArtifactRegistryRepositoryShouldFail(namespace, repoName, repoYAMLDir); err != nil {
   535  		t.Fatalf("error creating ArtifactRegistryRepository: %v", err)
   536  	}
   537  	ok, err := cluster.doesArtifactRegistryRepositoryHaveFinalizer(namespace, repoName, k8s.KCCFinalizer)
   538  	if err != nil {
   539  		t.Fatalf("error checking if ArtifactRegistryRepository has finalizer: %v", err)
   540  	}
   541  	if ok {
   542  		t.Fatalf("expected ArtifactRegistryRepository to not have finalizer '%v', but it does", k8s.KCCFinalizer)
   543  	}
   544  	ok, err = cluster.doesArtifactRegistryRepositoryHaveStatusUnmanaged(namespace, repoName, repoYAMLDir)
   545  	if err != nil {
   546  		t.Fatalf("error checking if ArtifactRegistryRepository has status '%v': %v", kcck8s.Unmanaged, err)
   547  	}
   548  	if !ok {
   549  		t.Fatalf("expected ArtifactRegistryRepository to have status '%v', but it does not", kcck8s.Unmanaged)
   550  	}
   551  	log.Info("Deleting ArtifactRegistryRepository...")
   552  	if err := cluster.deleteArtifactRegistryRepository(namespace, repoName); err != nil {
   553  		t.Fatal(err)
   554  	}
   555  }
   556  
   557  func TestUpgrade(t *testing.T) {
   558  	t.Parallel()
   559  	testOptions := newTestOptions()
   560  	testOptions.GKEClusterLocation = GKE_CLUSTER_ZONE2
   561  	testId, log, cluster, teardown := setup(t, testOptions)
   562  	if f.cleanup {
   563  		defer teardown()
   564  	}
   565  	//Get older version of the operator to perform an upgrade against
   566  	manifestsDir, sample, err := getOperatorReleaseAssetsForVersion(testOptions.BaseVersionSHA, testOptions.ServiceAccountID, testOptions.ProjectID, log)
   567  	if err != nil {
   568  		t.Fatalf("error getting operator release assets for version %v: %v", testOptions.BaseVersionSHA, err)
   569  	}
   570  	log.Info("Installing the base version operator...")
   571  	if err := cluster.installOperator(manifestsDir); err != nil {
   572  		t.Fatalf("error installing the base version operator: %v", err)
   573  	}
   574  	log.Info("Installing KCC...")
   575  	if err := cluster.installKCC(sample.configConnectorNamespacedModeYAMLPath); err != nil {
   576  		t.Fatalf("error installing KCC: %v", err)
   577  	}
   578  	namespace := "e2e-test-namespace"
   579  	if err := cluster.createNamespace(namespace); err != nil {
   580  		t.Fatalf("error creating namespace '%v': %v", namespace, err)
   581  	}
   582  	if err := cluster.enableKCCForNamespace(namespace, sample.configConnectorContextYAMLPath, testOptions.ServiceAccountID, testOptions.ProjectID); err != nil {
   583  		t.Fatalf("error enabling KCC for namespace '%v': %v", namespace, err)
   584  	}
   585  	if err := cluster.addProjectIDAnnotationToNamespace(namespace, f.projectID); err != nil {
   586  		t.Fatalf("error annotating namespace '%v' with the project ID: %v", namespace, err)
   587  	}
   588  	kccVersion, err := cluster.getKCCVersion()
   589  	if err != nil {
   590  		t.Fatalf("error determining KCC version: %v", err)
   591  	}
   592  	log.Info("Downloading and extracting KCC release tarball ...", "version", kccVersion)
   593  	kccReleaseAssetsDir, err := createTempDir("e2e-kcc-release-assets")
   594  	if err != nil {
   595  		t.Fatalf("error creating temporary directory for KCC release assets: %v", err)
   596  	}
   597  	if err := downloadAndExtractKCCReleaseTarball(kccVersion, kccReleaseAssetsDir); err != nil {
   598  		t.Fatalf("error downloading and extracting KCC with version '%v': %v", kccVersion, err)
   599  	}
   600  	repoName, repoYAMLDir, err := getArtifactRegistryRepositorySample(kccReleaseAssetsDir, testId)
   601  	if err != nil {
   602  		t.Fatalf("error getting ArtifactRegistryRepository sample from KCC release assets: %v", err)
   603  	}
   604  	log.Info("Creating ArtifactRegistryRepository...")
   605  	if err := cluster.createArtifactRegistryRepository(namespace, repoName, repoYAMLDir); err != nil {
   606  		t.Fatalf("error creating ArtifactRegistryRepository: %v", err)
   607  	}
   608  	log.Info("Upgrading the operator to the latest version")
   609  	manifestsDir, _, err = getOperatorReleaseAssetsForVersion(f.version, testOptions.ServiceAccountID, testOptions.ProjectID, log)
   610  	if err != nil {
   611  		t.Fatalf("error getting operator release assets for version '%v': %v", f.version, err)
   612  	}
   613  	log.Info("Installing the latest operator...")
   614  	if err := cluster.installOperator(manifestsDir); err != nil {
   615  		t.Fatalf("error installing the latest operator: %v", err)
   616  	}
   617  	time.Sleep(120 * time.Second) // Some buffer time for the operator to reconcile on the existing ConfigConnector
   618  	if err := cluster.waitForConfigConnectorToBeHealthy(k8s.ConfigConnectorAllowedName); err != nil {
   619  		t.Fatalf("error waitting for ConfigConnector to be healthy: %v", err)
   620  	}
   621  	checkIfKCCHasUpgradedToTheLatestVersion(t, cluster, log)
   622  	log.Info("Re-applying ArtifactRegistryRepository")
   623  	if err := cluster.createArtifactRegistryRepository(namespace, repoName, repoYAMLDir); err != nil {
   624  		t.Fatalf("error re-applying ArtifactRegistryRepository: %v", err)
   625  	}
   626  	log.Info("Deleting ArtifactRegistryRepository...")
   627  	if err := cluster.deleteArtifactRegistryRepository(namespace, repoName); err != nil {
   628  		t.Fatal(err)
   629  	}
   630  	log.Info("Deleting ConfigConnectorContext...")
   631  	if err := cluster.deleteConfigConnectorContext(namespace, k8s.ConfigConnectorContextAllowedName); err != nil {
   632  		t.Fatal(err)
   633  	}
   634  	log.Info("Uninstalling KCC...")
   635  	if err := cluster.uninstallKCC(); err != nil {
   636  		t.Fatalf("error uninstalling KCC: %v", err)
   637  	}
   638  }
   639  
   640  func newTestOptions() TestOptions {
   641  	return TestOptions{
   642  		OrganizationID:   organization,
   643  		BillingAccountID: billingAccount,
   644  		ServiceAccountID: SERVICE_ACC_ID,
   645  		BaseVersionSHA:   BASE_VERSION_SHA,
   646  		ProjectID:        f.projectID,
   647  	}
   648  }
   649  
   650  func checkIfKCCHasUpgradedToTheLatestVersion(t *testing.T, cluster *cluster, log logr.Logger) {
   651  	curKccVersionRaw, err := cluster.getKCCVersion()
   652  	if err != nil {
   653  		t.Fatalf("error getting the current KCC version: %v", err)
   654  	}
   655  	currentKccVersion, err := semver.ParseTolerant(curKccVersionRaw)
   656  	if err != nil {
   657  		t.Fatalf("current KCC version %v is not a valid semantic version", curKccVersionRaw)
   658  	}
   659  	latestOperatorVersionRaw, err := cluster.getOperatorVersion()
   660  	if err != nil {
   661  		t.Fatalf("error getting the latest operator version: %v", err)
   662  	}
   663  	latestOperatorVersion, err := semver.ParseTolerant(latestOperatorVersionRaw)
   664  	if err != nil {
   665  		t.Fatalf("latest Operator version %v is not a valid semantic version", curKccVersionRaw)
   666  	}
   667  	log.Info("Version checking", "currentKCCVersion", currentKccVersion, "latestOperatorVersion", latestOperatorVersion)
   668  	// only compare major.minor.patch as the operator version might contain operator specific extension e.g. 1.6.0-operator.x
   669  	if currentKccVersion.Major != latestOperatorVersion.Major || currentKccVersion.Minor != latestOperatorVersion.Minor || currentKccVersion.Patch != latestOperatorVersion.Patch {
   670  		t.Fatalf("expect to have KCC upgraded to %v, but it's still on version %v", latestOperatorVersion, currentKccVersion)
   671  	}
   672  }
   673  
   674  func setup(t *testing.T, testOptions TestOptions) (testId string, log logr.Logger, cluster *cluster, teardown func()) {
   675  	testId = newUniqueTestId()
   676  	log, err := newLogger(t.Name())
   677  	if err != nil {
   678  		t.Fatalf("error creating logger: %v", err)
   679  	}
   680  	clusterName := "e2e-test-" + testId
   681  	cluster, cleanup, err := setupCluster(clusterName, testOptions.ProjectID, testOptions.GKEClusterLocation,
   682  		testOptions.ServiceAccountID, log)
   683  	teardown = func() {
   684  		if cleanup != nil {
   685  			log.Info("Beginning cluster cleanup...")
   686  			if err := cleanup(); err != nil {
   687  				t.Errorf("error during cluster cleanup: %v", err)
   688  			}
   689  		}
   690  	}
   691  	if err != nil {
   692  		teardown()
   693  		t.Fatalf("error setting up cluster: %v", err)
   694  	}
   695  	if err := setupIdentity(testOptions, cluster.kubectl, log); err != nil {
   696  		teardown()
   697  		t.Fatalf("error setting up cluster: %v", err)
   698  	}
   699  	return testId, log, cluster, teardown
   700  }
   701  
   702  func cleanUpProject(deleteFunc cleanupFunc, shouldCleanUp bool, log logr.Logger) {
   703  	if shouldCleanUp {
   704  		log.Info("Beginning project cleanup...")
   705  		if err := deleteFunc(); err != nil {
   706  			log.Error(err, "error during project cleanup")
   707  		}
   708  	}
   709  }
   710  
   711  func getOperatorReleaseAssetsForVersion(version, serviceAccountID, projectID string, log logr.Logger) (manifestsDir string, sample configConnectorSample, err error) {
   712  	log.Info("Downloading and extracting operator release tarball...", "version", version)
   713  	emptySample := configConnectorSample{}
   714  	releaseAssetsDir, err := createTempDir("e2e-operator-release-assets")
   715  	if err != nil {
   716  		return "", emptySample, fmt.Errorf("error creating temporary directory for operator release assets: %v", err)
   717  	}
   718  	if err := downloadAndExtractOperatorReleaseTarball(version, releaseAssetsDir); err != nil {
   719  		return "", emptySample, fmt.Errorf("error downloading and extracting operator release tarball with version '%v': %v", version, err)
   720  	}
   721  	manifestsDir = path.Join(releaseAssetsDir, "operator-system")
   722  	sample, err = getConfigConnectorSample(releaseAssetsDir, serviceAccountID, projectID, version)
   723  	if err != nil {
   724  		return "", emptySample, fmt.Errorf("error getting ConfigConnector sample from operator release assets: %v", err)
   725  	}
   726  	return manifestsDir, sample, nil
   727  }
   728  
   729  func (c *cluster) installOperator(operatorManifestsDir string) error {
   730  	if _, err := c.kubectl.apply("-f", operatorManifestsDir); err != nil {
   731  		return fmt.Errorf("error applying operator manifests: %v", err)
   732  	}
   733  	time.Sleep(30 * time.Second) // Wait for the operator's controllers and webhooks to come up and be registered
   734  	return nil
   735  }
   736  
   737  func getConfigConnectorSample(operatorReleaseAssetsDir, serviceAccountID, projectID, version string) (sample configConnectorSample, err error) {
   738  	emptySample := configConnectorSample{}
   739  	samplesDir := path.Join(operatorReleaseAssetsDir, "samples")
   740  	var yamlPaths []string
   741  	sample = configConnectorSample{
   742  		configConnectorClusterModeWorkloadIdentityYAMLPath: path.Join(samplesDir, "configconnector_cluster_mode_workload_identity.yaml"),
   743  		configConnectorClusterModeGCPIdentityYAMLPath:      path.Join(samplesDir, "configconnector_cluster_mode_gcp_identity.yaml"),
   744  		configConnectorNamespacedModeYAMLPath:              path.Join(samplesDir, "configconnector_namespaced_mode.yaml"),
   745  		configConnectorContextYAMLPath:                     path.Join(samplesDir, "configconnectorcontext_sample.yaml"),
   746  	}
   747  	yamlPaths = []string{sample.configConnectorClusterModeWorkloadIdentityYAMLPath, sample.configConnectorNamespacedModeYAMLPath, sample.configConnectorContextYAMLPath}
   748  	for _, yamlPath := range yamlPaths {
   749  		content, err := ioutil.ReadFile(yamlPath)
   750  		if err != nil {
   751  			return emptySample, fmt.Errorf("error reading YAML: %v", err)
   752  		}
   753  		s := string(content)
   754  
   755  		// Replace vars (e.g. {GSA?})
   756  		vars := map[string]string{
   757  			"${GSA?}":        serviceAccountID,
   758  			"${PROJECT_ID?}": projectID,
   759  			"{GSA?}":         serviceAccountID,
   760  			"{PROJECT_ID?}":  projectID,
   761  		}
   762  		processOrder := []string{"${GSA?}", "${PROJECT_ID?}", "{GSA?}", "{PROJECT_ID?}"}
   763  		for _, k := range processOrder {
   764  			v := vars[k]
   765  			s = strings.ReplaceAll(s, k, v)
   766  		}
   767  
   768  		// Write back modified YAML to disk
   769  		if err := writeToFile(s, yamlPath); err != nil {
   770  			return emptySample, fmt.Errorf("error updating YAML file: %v", err)
   771  		}
   772  	}
   773  	return sample, nil
   774  }
   775  
   776  func (c *cluster) installKCC(configConnectorYAMLPath string) error {
   777  	content, err := ioutil.ReadFile(configConnectorYAMLPath)
   778  	if err != nil {
   779  		return fmt.Errorf("error reading ConfigConnector YAML: %v", err)
   780  	}
   781  	c.log.Info("Applying ConfigConnector YAML", "content", string(content))
   782  	if _, err := c.kubectl.apply("-f", configConnectorYAMLPath); err != nil {
   783  		return fmt.Errorf("error applying ConfigConnector YAML: %v", err)
   784  	}
   785  	if err := c.waitForConfigConnectorToBeHealthy(k8s.ConfigConnectorAllowedName); err != nil {
   786  		return err
   787  	}
   788  
   789  	// Wait for KCC's components to come up and be registered
   790  	return c.waitForAllComponentPodsReady()
   791  }
   792  
   793  func (c *cluster) enableKCCForNamespace(namespace, configConnectorContextYAMLPath, serviceAccountId, projectID string) error {
   794  	c.log.Info("Setting up Workload Identity binding for namespace...", "namespace", namespace)
   795  	serviceAccEmail := fmt.Sprintf("%v@%v.iam.gserviceaccount.com", serviceAccountId, projectID)
   796  	if err := setupWorkloadIdentityForNamespace(namespace, serviceAccEmail, projectID); err != nil {
   797  		return fmt.Errorf("error setting up Workload Identity binding for namespace '%v': %v", namespace, err)
   798  	}
   799  
   800  	content, err := ioutil.ReadFile(configConnectorContextYAMLPath)
   801  	if err != nil {
   802  		return fmt.Errorf("error reading ConfigConnectorContext YAML for namespace '%v': %v", namespace, err)
   803  	}
   804  	c.log.Info("Applying ConfigConnectorContext YAML", "namespace", namespace, "content", string(content))
   805  	if _, err := c.kubectl.apply("-n", namespace, "-f", configConnectorContextYAMLPath); err != nil {
   806  		return fmt.Errorf("error applying ConfigConnectorContext YAML for namespace '%v': %v", namespace, err)
   807  	}
   808  	if err := c.waitForConfigConnectorToBeHealthy(k8s.ConfigConnectorAllowedName); err != nil {
   809  		return err
   810  	}
   811  	if err := c.waitForConfigConnectorContextToBeHealthy(namespace, k8s.ConfigConnectorContextAllowedName); err != nil {
   812  		return err
   813  	}
   814  	time.Sleep(90 * time.Second) // Wait for a KCC controller to come up and be registered for the given namespace
   815  	return nil
   816  }
   817  
   818  func (c *cluster) waitForConfigConnectorToBeHealthy(name string) error {
   819  	err := wait.PollImmediate(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   820  		f := func() (interface{}, error) {
   821  			return c.kubectl.get("configconnector", name, "-o", "yaml")
   822  		}
   823  		res, err := c.retry(f, longIntervalBackOff)
   824  		if err != nil {
   825  			return false, fmt.Errorf("error getting ConfigConnector '%v': %v", name, err)
   826  		}
   827  		c.log.Info("Waiting for ConfigConnector to reach a healthy state...", "name", name)
   828  		return strings.Contains(res.(string), "healthy: true"), nil
   829  	})
   830  	if err != nil {
   831  		if err != wait.ErrWaitTimeout {
   832  			return err
   833  		}
   834  		out, _ := c.kubectl.get("configconnector", name, "-o", "yaml")
   835  		return fmt.Errorf("timed out waiting for ConfigConnector '%v' to reach a healthy state:\n%v", name, out)
   836  	}
   837  	c.log.Info("ConfigConnector has reached a healthy state", "name", name)
   838  	return nil
   839  }
   840  
   841  func (c *cluster) waitForAllComponentPodsReady() error {
   842  	c.log.Info("waiting for all component pods in 'cnrm-system' namespace to be ready...")
   843  	out, err := c.kubectl.exec("wait", "", "-n", "cnrm-system", "--for=condition=Ready", "pod", "--all", "--timeout=180s")
   844  	if err != nil {
   845  		return fmt.Errorf("error waiting for all component pods in 'cnrm-system' namespace to be ready: %w, output: %v", err, out)
   846  	}
   847  	return nil
   848  }
   849  
   850  func (c *cluster) waitForConfigConnectorContextToBeRemoved(namespace, name string) error {
   851  	err := wait.PollImmediate(5*time.Second, 5*time.Minute, func() (done bool, err error) {
   852  		var isDeleted bool
   853  		f := func() (interface{}, error) {
   854  			c.log.Info("Getting ConfigConnectorContext...", "namespace", namespace, "name", name)
   855  			out, err := c.kubectl.get("-n", namespace, "configconnectorcontext", name)
   856  			if err != nil {
   857  				// Quick exit if the ConfigConnectorContext object is deleted.
   858  				if strings.Contains(err.Error(), "Error from server (NotFound)") {
   859  					isDeleted = true
   860  					return nil, nil
   861  				}
   862  				return nil, err
   863  			}
   864  			return out, nil
   865  		}
   866  		out, err := c.retry(f, defaultBackOff)
   867  		if err != nil {
   868  			return false, fmt.Errorf("unexpected error getting ConfigConnectorContext: %w; command output: %v", err, out)
   869  		}
   870  		if isDeleted {
   871  			return true, nil
   872  		}
   873  		c.log.Info("Waiting for ConfigConnectorContext to be deleted...", "namespace", namespace, "name", name, "output", out)
   874  		return false, nil
   875  	})
   876  	if err == nil {
   877  		return nil
   878  	}
   879  	if err != wait.ErrWaitTimeout {
   880  		return err
   881  	}
   882  	out, _ := c.kubectl.get("-n", namespace, "configconnectorcontext", name, "-o", "yaml")
   883  	return fmt.Errorf("timed out waiting for ConfigConnectorContext '%v/%v' to be removed:\n%v", namespace, name, out)
   884  }
   885  
   886  func (c *cluster) waitForConfigConnectorContextToBeHealthy(namespace, name string) error {
   887  	return c.waitForConfigConnectorContextToBeHealthyOrUnhealthy(namespace, name, true)
   888  }
   889  
   890  func (c *cluster) waitForConfigConnectorContextToBeUnhealthy(namespace, name string) error {
   891  	return c.waitForConfigConnectorContextToBeHealthyOrUnhealthy(namespace, name, false)
   892  }
   893  
   894  func (c *cluster) waitForConfigConnectorContextToBeHealthyOrUnhealthy(namespace, name string, healthy bool) error {
   895  	desiredState := "unhealthy"
   896  	if healthy {
   897  		desiredState = "healthy"
   898  	}
   899  	err := wait.PollImmediate(5*time.Second, 10*time.Minute, func() (done bool, err error) {
   900  		f := func() (interface{}, error) {
   901  			return c.kubectl.get("-n", namespace, "configconnectorcontext", name, "-o", "yaml")
   902  		}
   903  		out, err := c.retry(f, longIntervalBackOff)
   904  		if err != nil {
   905  			return false, fmt.Errorf("error getting ConfigConnectorContext '%v' for namespace '%v': %v", name, namespace, err)
   906  		}
   907  		c.log.Info(fmt.Sprintf("Waiting for ConfigConnectorContext to reach an %v state...", desiredState), "namespace", namespace, "name", name)
   908  		return strings.Contains(out.(string), fmt.Sprintf("healthy: %v", healthy)), nil
   909  	})
   910  	if err != nil {
   911  		if err != wait.ErrWaitTimeout {
   912  			return err
   913  		}
   914  		out, _ := c.kubectl.get("-n", namespace, "configconnectorcontext", name, "-o", "yaml")
   915  		return fmt.Errorf("timed out waiting for ConfigConnectorContext '%v/%v' to reach an %v state:\n%v", namespace, name, desiredState, out)
   916  	}
   917  	c.log.Info(fmt.Sprintf("ConfigConnectorContext has reached an %v state", desiredState), "namespace", namespace, "name", name)
   918  	return nil
   919  }
   920  
   921  func setupWorkloadIdentityForNamespace(namespace, serviceAccEmail, projectID string) error {
   922  	member := fmt.Sprintf("serviceAccount:%v.svc.id.goog[cnrm-system/%v%v]", projectID, k8s.ServiceAccountNamePrefix, namespace)
   923  	role := "roles/iam.workloadIdentityUser"
   924  	if err := addIAMBindingForServiceAcc(serviceAccEmail, member, role, projectID); err != nil {
   925  		return fmt.Errorf("error setting up Workload Identity binding: %v", err)
   926  	}
   927  	return nil
   928  }
   929  
   930  func getArtifactRegistryRepositorySample(kccReleaseAssetsDir, uniqueId string) (repoName string, repoYAMLDir string, err error) {
   931  	repoYAMLDir = path.Join(kccReleaseAssetsDir, "samples", "resources", "artifactregistryrepository")
   932  	yamlPaths, err := getYAMLFilesInDir(repoYAMLDir)
   933  	if err != nil {
   934  		return "", "", fmt.Errorf("error getting paths to YAML files in ArtifactRegistryRepository sample directory '%v': %v", repoYAMLDir, err)
   935  	}
   936  	for _, yamlPath := range yamlPaths {
   937  		b, err := ioutil.ReadFile(yamlPath)
   938  		if err != nil {
   939  			return "", "", fmt.Errorf("error reading file '%v': %v", yamlPath, err)
   940  		}
   941  		s := string(b)
   942  		s = strings.ReplaceAll(s, "sample", "sample"+uniqueId)
   943  		s = strings.ReplaceAll(s, "dep", "dep"+uniqueId)
   944  
   945  		// Write back modified YAML to disk
   946  		if err := writeToFile(s, yamlPath); err != nil {
   947  			return "", "", fmt.Errorf("error updating file '%v': %v", yamlPath, err)
   948  		}
   949  	}
   950  	repoName, err = getRepoNameFromArtifactRegistryRepositorySampleDir(repoYAMLDir)
   951  	if err != nil {
   952  		return "", "", fmt.Errorf("error getting name of ArtifactRegistryRepository for ArtifactRegistryRepository sample directory '%v': %v", repoYAMLDir, err)
   953  	}
   954  	return repoName, repoYAMLDir, nil
   955  }
   956  
   957  func getRepoNameFromArtifactRegistryRepositorySampleDir(repoYAMLDir string) (string, error) {
   958  	unstructs := make([]*unstructured.Unstructured, 0)
   959  	yamlPaths, err := getYAMLFilesInDir(repoYAMLDir)
   960  	if err != nil {
   961  		return "", fmt.Errorf("error getting paths to YAML files in directory '%v': %v", repoYAMLDir, err)
   962  	}
   963  	for _, yamlPath := range yamlPaths {
   964  		u, err := utils.ReadFileToUnstructs(yamlPath)
   965  		if err != nil {
   966  			return "", fmt.Errorf("error converting file '%v' to unstructs: %v", yamlPath, err)
   967  		}
   968  		unstructs = append(unstructs, u...)
   969  	}
   970  	repoNames := make([]string, 0)
   971  	for _, u := range unstructs {
   972  		if u.GetKind() == "ArtifactRegistryRepository" {
   973  			repoNames = append(repoNames, u.GetName())
   974  		}
   975  	}
   976  	switch len(repoNames) {
   977  	case 0:
   978  		return "", fmt.Errorf("no ArtifactRegistryRepository found in directory '%v'", repoYAMLDir)
   979  	case 1:
   980  		return repoNames[0], nil
   981  	default:
   982  		return "", fmt.Errorf("multiple ArtifactRegistryRepositories found in directory '%v'", repoYAMLDir)
   983  	}
   984  }
   985  
   986  func getYAMLFilesInDir(dir string) (yamlPaths []string, err error) {
   987  	yamlPaths = make([]string, 0)
   988  	fileInfos, err := ioutil.ReadDir(dir)
   989  	if err != nil {
   990  		return []string{}, fmt.Errorf("error reading directory '%v': %v", dir, err)
   991  	}
   992  	for _, fi := range fileInfos {
   993  		if fi.IsDir() {
   994  			continue
   995  		}
   996  		if !strings.HasSuffix(fi.Name(), ".yaml") {
   997  			continue
   998  		}
   999  		yamlPaths = append(yamlPaths, path.Join(dir, fi.Name()))
  1000  	}
  1001  	return yamlPaths, nil
  1002  }
  1003  
  1004  func (c *cluster) createArtifactRegistryRepository(namespace, repoName, repoYAMLDir string) error {
  1005  	if err := c.createArtifactRegistryRepositoryAndWait(namespace, repoName, repoYAMLDir); err != nil {
  1006  		if err == wait.ErrWaitTimeout {
  1007  			out, _ := c.kubectl.get("-n", namespace, "artifactregistryrepository", repoName, "-o", "yaml")
  1008  			return fmt.Errorf("timed out waiting for ArtifactRegistryRepository to reach an UpToDate state:\n%v", out)
  1009  		}
  1010  		return err
  1011  	}
  1012  	return nil
  1013  }
  1014  
  1015  func (c *cluster) createArtifactRegistryRepositoryShouldFail(namespace, repoName, repoYAMLDir string) error {
  1016  	if err := c.createArtifactRegistryRepositoryAndWait(namespace, repoName, repoYAMLDir); err != nil {
  1017  		if err == wait.ErrWaitTimeout {
  1018  			return nil // i.e. ArtifactRegistryRepository never reached an "UpToDate" state as expected
  1019  		}
  1020  		return err
  1021  	}
  1022  	// ArtifactRegistryRepository ended up being created successfully contrary to expectations, so return an error
  1023  	out, _ := c.kubectl.get("-n", namespace, "artifactregistryrepository", repoName, "-o", "yaml")
  1024  	return fmt.Errorf("expected creation of ArtifactRegistryRepository to fail, but got:\n%v", out)
  1025  }
  1026  
  1027  func (c *cluster) createArtifactRegistryRepositoryAndWait(namespace, repoName, repoYAMLDir string) error {
  1028  	if _, err := c.kubectl.apply("-n", namespace, "-f", repoYAMLDir); err != nil {
  1029  		return fmt.Errorf("error applying ArtifactRegistryRepository: %v", err)
  1030  	}
  1031  	return wait.PollImmediate(5*time.Second, 2*time.Minute, func() (done bool, err error) {
  1032  		c.log.Info("Getting ArtifactRegistryRepository...", "name", repoName)
  1033  		f := func() (interface{}, error) {
  1034  			return c.kubectl.get("-n", namespace, "artifactregistryrepository", repoName, "-o", "yaml")
  1035  		}
  1036  		// Sometime, polling on object returns some transient-ish connection errors;
  1037  		// here we want to be more tolerant/robust by retrying a little more with a longer interval.
  1038  		out, err := c.retry(f, longIntervalBackOff)
  1039  		if err != nil {
  1040  			return false, fmt.Errorf("error getting ArtifactRegistryRepository '%v/%v': %v", namespace, repoName, err)
  1041  		}
  1042  		c.log.Info("Waiting for ArtifactRegistryRepository to reach an UpToDate state...", "name", repoName)
  1043  		return strings.Contains(out.(string), "UpToDate"), nil
  1044  	})
  1045  }
  1046  
  1047  func (c *cluster) waitForCNRMFinalizersToBeRemovedFromArtifactRegistryRepository(namespace, repoName string) error {
  1048  	waitFunc := func() (done bool, err error) {
  1049  		ok, err := c.doesArtifactRegistryRepositoryHaveFinalizer(namespace, repoName, k8s.KCCFinalizer)
  1050  		if err != nil {
  1051  			return false, fmt.Errorf("error checking for the finalizer on ArtifactRegistryRepository: %w", err)
  1052  		}
  1053  		return !ok, nil
  1054  	}
  1055  	if err := wait.PollImmediate(5*time.Second, 5*time.Minute, waitFunc); err != nil {
  1056  		if err != wait.ErrWaitTimeout {
  1057  			return err
  1058  		}
  1059  		out, _ := c.kubectl.get("-n", namespace, "artifactregistryrepository", repoName, "-o", "yaml")
  1060  		return fmt.Errorf("timed out waiting for the CNRM finalizers to be removed from ArtifactRegistryRepository '%v/%v':\n%v", namespace, repoName, out)
  1061  	}
  1062  	return nil
  1063  }
  1064  
  1065  func (c *cluster) getArtifactRegistryRepositoryUnstructured(namespace, repoName string) (*unstructured.Unstructured, error) {
  1066  	out, err := c.kubectl.get("-n", namespace, "artifactregistryrepository", repoName, "-o", "yaml")
  1067  	if err != nil {
  1068  		return nil, fmt.Errorf("error getting ArtifactRegistryRepository '%v/%v': %v", namespace, repoName, err)
  1069  	}
  1070  	repoUnstruct, err := utils.BytesToUnstruct([]byte(out))
  1071  	if err != nil {
  1072  		return nil, fmt.Errorf("error converting '%v' to unstruct: %w", out, err)
  1073  	}
  1074  	return repoUnstruct, err
  1075  }
  1076  
  1077  func (c *cluster) applyUnstructured(u *unstructured.Unstructured) error {
  1078  	bytes, err := utils.UnstructToYaml(u)
  1079  	if err != nil {
  1080  		return fmt.Errorf("error converting unstruct to yaml: %w", err)
  1081  	}
  1082  	if _, err := c.kubectl.applyStdin(string(bytes), "-f", "-"); err != nil {
  1083  		return fmt.Errorf("error applying %v after adding finalizer: %w", u.GetKind(), err)
  1084  	}
  1085  	return nil
  1086  }
  1087  
  1088  func (c *cluster) removeFinalizerToArtifactRegistryRepository(namespace, repoName, finalizer string) error {
  1089  	repoUnstruct, err := c.getArtifactRegistryRepositoryUnstructured(namespace, repoName)
  1090  	if err != nil {
  1091  		return err
  1092  	}
  1093  	finalizers := append(repoUnstruct.GetFinalizers(), finalizer)
  1094  	var newFinalizers []string
  1095  	for _, f := range finalizers {
  1096  		if f == finalizer {
  1097  			continue
  1098  		}
  1099  		newFinalizers = append(newFinalizers, f)
  1100  	}
  1101  	repoUnstruct.SetFinalizers(newFinalizers)
  1102  	return c.applyUnstructured(repoUnstruct)
  1103  }
  1104  
  1105  func (c *cluster) addFinalizerToArtifactRegistryRepository(namespace, repoName, finalizer string) error {
  1106  	repoUnstruct, err := c.getArtifactRegistryRepositoryUnstructured(namespace, repoName)
  1107  	if err != nil {
  1108  		return err
  1109  	}
  1110  	finalizers := append(repoUnstruct.GetFinalizers(), finalizer)
  1111  	repoUnstruct.SetFinalizers(finalizers)
  1112  	return c.applyUnstructured(repoUnstruct)
  1113  }
  1114  
  1115  func (c *cluster) doesArtifactRegistryRepositoryHaveFinalizer(namespace, repoName, finalizer string) (ok bool, err error) {
  1116  	repoUnstruct, err := c.getArtifactRegistryRepositoryUnstructured(namespace, repoName)
  1117  	if err != nil {
  1118  		return false, err
  1119  	}
  1120  	for _, f := range repoUnstruct.GetFinalizers() {
  1121  		if finalizer == f {
  1122  			return true, nil
  1123  		}
  1124  	}
  1125  	return false, nil
  1126  }
  1127  
  1128  func (c *cluster) doesArtifactRegistryRepositoryHaveStatusUnmanaged(namespace, repoName, finalizer string) (ok bool, err error) {
  1129  	repoUnstruct, err := c.getArtifactRegistryRepositoryUnstructured(namespace, repoName)
  1130  	if err != nil {
  1131  		return false, err
  1132  	}
  1133  	r, err := kcck8s.NewResource(repoUnstruct)
  1134  	if err != nil {
  1135  		return false, err
  1136  	}
  1137  	condition, ok := kcck8s.GetReadyCondition(r)
  1138  	if !ok {
  1139  		return false, nil
  1140  	}
  1141  	return condition.Reason == kcck8s.Unmanaged, nil
  1142  }
  1143  
  1144  func (c *cluster) deleteArtifactRegistryRepository(namespace, repoName string, extraArgs ...string) error {
  1145  	args := []string{"-n", namespace, "artifactregistryrepository", repoName}
  1146  	args = append(args, extraArgs...)
  1147  	f := func() (interface{}, error) {
  1148  		return c.kubectl.delete(args...)
  1149  	}
  1150  	_, err := c.retry(f, defaultBackOff)
  1151  	if err != nil {
  1152  		return fmt.Errorf("error deleting ArtifactRegistryRepository: %v", err)
  1153  	}
  1154  	return nil
  1155  }
  1156  
  1157  func (c *cluster) deleteConfigConnectorContext(namespace, name string) error {
  1158  	args := []string{"-n", namespace, "configconnectorcontext", name}
  1159  	f := func() (interface{}, error) {
  1160  		if _, err := c.kubectl.delete(args...); err != nil {
  1161  			c.log.Info("error deleting ConfigConnectorContext...", "error", err)
  1162  			return nil, err
  1163  		}
  1164  		return nil, nil
  1165  	}
  1166  	if _, err := c.retry(f, longIntervalBackOff); err != nil {
  1167  		return fmt.Errorf("error deleting ConfigConnectorContext: %v", err)
  1168  	}
  1169  	return c.waitForConfigConnectorContextToBeRemoved(namespace, name)
  1170  }
  1171  
  1172  func (c *cluster) uninstallKCC() error {
  1173  	c.log.Info("deleting the ConfigConnector object")
  1174  	f := func() (interface{}, error) {
  1175  		if _, err := c.kubectl.delete("configconnector", k8s.ConfigConnectorAllowedName); err != nil {
  1176  			c.log.Info("error deleting ConfigConnector...", "error", err)
  1177  			return nil, err
  1178  		}
  1179  		return nil, nil
  1180  	}
  1181  	if _, err := c.retry(f, longIntervalBackOff); err != nil {
  1182  		return fmt.Errorf("error deleting ConfigConnectors: %v", err)
  1183  	}
  1184  	c.log.Info("Asserting that the ConfigConnector object is gone")
  1185  	out, err := c.kubectl.get("configconnector")
  1186  	if err != nil {
  1187  		return fmt.Errorf("error getting ConfigConnectors: %v", err)
  1188  	}
  1189  	if !strings.Contains(out, "No resources found") {
  1190  		return fmt.Errorf("expected no ConfigConnectors to exist, but got:\n%v", out)
  1191  	}
  1192  
  1193  	// As the uninstallation is no longer blocked by the deletion of the ignored
  1194  	// CRDs, the following assertion might fail in the beginning. But the ignored
  1195  	// CRDs have the ownerReferences to the ConfigConnector object, so after the
  1196  	// ConfigConnector object is deleted, the ignored CRDs will eventually be garbage
  1197  	// collected.
  1198  	// Retrying the assertion to simulate the latest UX.
  1199  	crdAssertionFunc := func() (interface{}, error) {
  1200  		c.log.Info("Asserting that resource CRDs are deleted")
  1201  		out, err = c.kubectl.get("crds", "--selector", "cnrm.cloud.google.com/managed-by-kcc=true")
  1202  		if err != nil {
  1203  			return nil, fmt.Errorf("error getting KCC CRDs: %v", err)
  1204  		}
  1205  		if !strings.Contains(out, "No resources found") {
  1206  			return nil, fmt.Errorf("expected KCC CRDs to not exist, but got:\n%v", out)
  1207  		}
  1208  		return out, nil
  1209  	}
  1210  	_, err = c.retry(crdAssertionFunc, defaultBackOff)
  1211  	if err != nil {
  1212  		return fmt.Errorf("unexpected error asserting that resource CRDs are deleted: %w", err)
  1213  	}
  1214  
  1215  	out, err = c.kubectl.get("validatingwebhookconfiguration")
  1216  	if err != nil {
  1217  		return fmt.Errorf("error getting ValidatingWebhookConfigurations: %v", err)
  1218  	}
  1219  	if strings.Contains(out, k8s.CNRMDomain) {
  1220  		return fmt.Errorf("expected KCC validating webhooks to not exist, but got:\n%v", out)
  1221  	}
  1222  	out, err = c.kubectl.get("mutatingwebhookconfiguration")
  1223  	if err != nil {
  1224  		return fmt.Errorf("error getting MutatingWebhookConfigurations: %v", err)
  1225  	}
  1226  	if strings.Contains(out, k8s.CNRMDomain) {
  1227  		return fmt.Errorf("expected KCC mutating webhooks to not exist, but got:\n%v", out)
  1228  	}
  1229  	c.log.Info("Asserting that `cnrm-system` namespace is deleted")
  1230  	return c.waitForNamespaceToBeDeleted(k8s.CNRMSystemNamespace)
  1231  }
  1232  
  1233  func (c *cluster) waitForNamespaceToBeDeleted(namespace string) error {
  1234  	// Deleting a namespace can take a long time. Give some buffer time for k8s api server to process.
  1235  	time.Sleep(5 * time.Minute)
  1236  
  1237  	err := wait.PollImmediate(20*time.Second, 10*time.Minute, func() (done bool, err error) {
  1238  		c.log.Info("Getting namespace...", "namespace", namespace)
  1239  		var isDeleted bool
  1240  		f := func() (interface{}, error) {
  1241  			ns, err := c.getNamespace(namespace)
  1242  			if err != nil {
  1243  				// Quick exit if the namespace is deleted already.
  1244  				if errors.IsNotFound(err) {
  1245  					isDeleted = true
  1246  					return nil, nil
  1247  				}
  1248  				return nil, err
  1249  			}
  1250  			return ns, nil
  1251  		}
  1252  		// Sometime, polling on object returns some transient-ish connection errors;
  1253  		// here we want to be more tolerant/robust by retrying a little more with a longer interval.
  1254  		res, err := c.retry(f, longIntervalBackOff)
  1255  		if err != nil {
  1256  			return false, fmt.Errorf("error getting namespace '%v': %v", namespace, err)
  1257  		}
  1258  		if isDeleted {
  1259  			return true, nil
  1260  		}
  1261  		ns := res.(*v1.Namespace)
  1262  		c.log.Info("Waiting for namespace to be deleted...", "namespace", namespace, "status", ns.Status)
  1263  		return false, nil
  1264  	})
  1265  	if err != nil {
  1266  		if err != wait.ErrWaitTimeout {
  1267  			return err
  1268  		}
  1269  		return fmt.Errorf("timed out waiting for namespace '%v' to be deleted", namespace)
  1270  	}
  1271  	return nil
  1272  }
  1273  
  1274  func checkArtifactRegistryRepositoryExistsOnGCP(repoName, projectID string) error {
  1275  	cmd := exec.Command("gcloud", "artifacts", "repositories", "describe", repoName,
  1276  		"--location", "us-west1",
  1277  		"--project", projectID)
  1278  	_, err := utils.ExecuteAndCaptureOutput(cmd)
  1279  	if err != nil {
  1280  		if strings.Contains(err.Error(), "NOT_FOUND") {
  1281  			return fmt.Errorf("expected project '%v' to have Artifact Registry Repository named '%v', but got:\n%v", projectID, repoName, err)
  1282  		}
  1283  		return fmt.Errorf("error checking if Artifact Registry Repository exists on GCP: %v", err)
  1284  	}
  1285  	return nil
  1286  }
  1287  
  1288  func setupCluster(clusterName, projectID, location, serviceAccountId string, log logr.Logger) (*cluster, cleanupFunc, error) {
  1289  	var cleanup cleanupFunc
  1290  	log.Info("Creating a Container client...")
  1291  	ctx := context.Background()
  1292  	container, err := containerBeta.NewService(ctx)
  1293  	if err != nil {
  1294  		return nil, cleanup, fmt.Errorf("error creating Container client: %v", err)
  1295  	}
  1296  	log.Info("Creating a GKE cluster...", "name", clusterName)
  1297  	if err := createGKECluster(container, clusterName, projectID, location, log); err != nil {
  1298  		return nil, cleanup, fmt.Errorf("error creating GKE cluster with name '%v': %v", clusterName, err)
  1299  	}
  1300  	cleanup = func() error {
  1301  		log.Info("Deleting GKE cluster...", "name", clusterName)
  1302  		if err := deleteGKECluster(container, clusterName, projectID, location); err != nil {
  1303  			return fmt.Errorf("error deleting cluster with name '%v': %v", clusterName, err)
  1304  		}
  1305  		return nil
  1306  	}
  1307  
  1308  	log.Info("Getting the cluster's kubeconfig...")
  1309  	outDirForKubeconfig, err := createTempDir("e2e-" + clusterName + "-kubeconfig")
  1310  	if err != nil {
  1311  		return nil, cleanup, fmt.Errorf("error creating temporary directory for cluster's kubeconfig: %v", err)
  1312  	}
  1313  	outPathForKubeconfig := path.Join(outDirForKubeconfig, "kubeconfig.yaml")
  1314  	if err := getKubeconfigForGKECluster(projectID, location, clusterName, outPathForKubeconfig); err != nil {
  1315  		return nil, cleanup, fmt.Errorf("error getting cluster's kubeconfig: %v", err)
  1316  	}
  1317  	log.Info("Setting up a client-go Clientset...")
  1318  	config, err := clientcmd.BuildConfigFromFlags("", outPathForKubeconfig)
  1319  	if err != nil {
  1320  		return nil, cleanup, fmt.Errorf("error building REST client config from kubeconfig: %v", err)
  1321  	}
  1322  	clientset, err := kubernetes.NewForConfig(config)
  1323  	if err != nil {
  1324  		return nil, cleanup, fmt.Errorf("error creating client-go Clientset: %v", err)
  1325  	}
  1326  	cluster := &cluster{
  1327  		kubectl: &kubectl{
  1328  			kubeconfigPath: outPathForKubeconfig,
  1329  			deleteTimeout:  KUBECTL_DELETE_TIMEOUT,
  1330  		},
  1331  		clientset: clientset,
  1332  		log:       log,
  1333  	}
  1334  	return cluster, cleanup, nil
  1335  }
  1336  
  1337  func setupIdentity(testOptions TestOptions, k *kubectl, log logr.Logger) error {
  1338  	if testOptions.SecretName != "" {
  1339  		log.Info("Creating a secret containing service account key..")
  1340  		serviceAccEmail := fmt.Sprintf("%v@%v.iam.gserviceaccount.com", testOptions.ServiceAccountID, testOptions.ProjectID)
  1341  		if err := createCredentialSecret(serviceAccEmail, testOptions.ProjectID, testOptions.SecretName, k); err != nil {
  1342  			return err
  1343  		}
  1344  		return nil
  1345  	} else {
  1346  		log.Info("Setting up Workload Identity binding...")
  1347  		serviceAccEmail := fmt.Sprintf("%v@%v.iam.gserviceaccount.com", testOptions.ServiceAccountID, testOptions.ProjectID)
  1348  		member := fmt.Sprintf("serviceAccount:%v.svc.id.goog[cnrm-system/cnrm-controller-manager]", testOptions.ProjectID)
  1349  		role := "roles/iam.workloadIdentityUser"
  1350  		if err := addIAMBindingForServiceAcc(serviceAccEmail, member, role, testOptions.ProjectID); err != nil {
  1351  			return err
  1352  		}
  1353  		return nil
  1354  	}
  1355  }
  1356  
  1357  func setupProject(organizationID, projectID, billingAccountID, serviceAccountID string, log logr.Logger) (cleanupFunc, error) {
  1358  	var cleanup cleanupFunc
  1359  	log.Info("Creating GCP clients...")
  1360  	ctx := context.Background()
  1361  	resourceManagerClient, err := cloudresourcemanager.NewService(ctx)
  1362  	if err != nil {
  1363  		return nil, fmt.Errorf("error creating ResourceManager client: %v", err)
  1364  	}
  1365  	billingClient, err := cloudbilling.NewService(ctx)
  1366  	if err != nil {
  1367  		return nil, fmt.Errorf("error creating Billing client: %v", err)
  1368  	}
  1369  	iamClient, err := iam.NewService(ctx)
  1370  	if err != nil {
  1371  		return nil, fmt.Errorf("error creating IAM client: %v", err)
  1372  	}
  1373  	log.Info("Creating project...", "projectID", projectID)
  1374  	if err := createProject(resourceManagerClient, organizationID, projectID, log); err != nil {
  1375  		return cleanup, fmt.Errorf("error creating project with project ID '%v': %v", projectID, err)
  1376  	}
  1377  	cleanup = func() error {
  1378  		log.Info("Deleting project...", "projectID", projectID)
  1379  		if err := deleteProject(resourceManagerClient, projectID); err != nil {
  1380  			return fmt.Errorf("error deleting project with project ID '%v': %v", projectID, err)
  1381  		}
  1382  		return nil
  1383  	}
  1384  	log.Info("Linking project to billing account...", "billingAccount", billingAccountID)
  1385  	if err := linkProjectToBillingAccount(billingClient, projectID, billingAccountID); err != nil {
  1386  		return cleanup, fmt.Errorf("error linking project to billing account '%v'", billingAccountID)
  1387  	}
  1388  	log.Info("Enabling services for project...")
  1389  	if err := enableServicesForProject(projectID, SERVICES, log); err != nil {
  1390  		return cleanup, fmt.Errorf("error enabling services for project: %v", err)
  1391  	}
  1392  	log.Info("Setting up IAM service account...")
  1393  	if err := createServiceAccount(iamClient, serviceAccountID, projectID); err != nil {
  1394  		return cleanup, fmt.Errorf("error creating service account: %v", err)
  1395  	}
  1396  	serviceAccEmail := fmt.Sprintf("%v@%v.iam.gserviceaccount.com", serviceAccountID, projectID)
  1397  	if err := addIAMBindingForProject(projectID, "serviceAccount:"+serviceAccEmail, "roles/owner"); err != nil {
  1398  		return cleanup, fmt.Errorf("error granting service account project owner role: %v", err)
  1399  	}
  1400  	return cleanup, nil
  1401  }
  1402  
  1403  func createProject(resourceManagerClient *cloudresourcemanager.Service, organizationID, projectID string, log logr.Logger) error {
  1404  	project := &cloudresourcemanager.Project{
  1405  		ProjectId: projectID,
  1406  		Labels: map[string]string{
  1407  			"cnrm-test": "true",
  1408  		},
  1409  		Parent: &cloudresourcemanager.ResourceId{
  1410  			Type: "organization",
  1411  			Id:   organizationID,
  1412  		},
  1413  	}
  1414  	op, err := resourceManagerClient.Projects.Create(project).Do()
  1415  	if err != nil {
  1416  		return err
  1417  	}
  1418  	// Wait for project creation operation to finish
  1419  	return wait.PollImmediate(5*time.Second, 5*time.Minute, func() (done bool, err error) {
  1420  		res, err := resourceManagerClient.Operations.Get(op.Name).Do()
  1421  		if err != nil {
  1422  			return false, err
  1423  		}
  1424  		if res.Done {
  1425  			if res.Error != nil {
  1426  				return true, fmt.Errorf("project creation operation failed: %v", res.Error)
  1427  			}
  1428  			return true, nil
  1429  		}
  1430  		log.Info("Waiting for project creation operation to finish...")
  1431  		return false, nil
  1432  	})
  1433  }
  1434  
  1435  func deleteProject(resourceManagerClient *cloudresourcemanager.Service, projectID string) error {
  1436  	_, err := resourceManagerClient.Projects.Delete(projectID).Do()
  1437  	if err != nil {
  1438  		return err
  1439  	}
  1440  	return nil
  1441  }
  1442  
  1443  func linkProjectToBillingAccount(billingClient *cloudbilling.APIService, projectID, billingAccount string) error {
  1444  	ba := &cloudbilling.ProjectBillingInfo{
  1445  		BillingAccountName: "billingAccounts/" + billingAccount,
  1446  	}
  1447  	_, err := billingClient.Projects.UpdateBillingInfo("projects/"+projectID, ba).Do()
  1448  	return err
  1449  }
  1450  
  1451  func enableServicesForProject(projectID string, services []string, log logr.Logger) error {
  1452  	for _, service := range services {
  1453  		log.Info("Enabling service...", "service", service)
  1454  		cmd := exec.Command("gcloud", "services", "enable", service, "--project", projectID)
  1455  		if err := utils.Execute(cmd); err != nil {
  1456  			return err
  1457  		}
  1458  	}
  1459  	return nil
  1460  }
  1461  
  1462  func createServiceAccount(iamClient *iam.Service, serviceAccountID, projectID string) error {
  1463  	req := &iam.CreateServiceAccountRequest{
  1464  		AccountId: serviceAccountID,
  1465  	}
  1466  	_, err := iamClient.Projects.ServiceAccounts.Create("projects/"+projectID, req).Do()
  1467  	if err != nil {
  1468  		return err
  1469  	}
  1470  	return nil
  1471  }
  1472  
  1473  func addIAMBindingForProject(projectID, member, role string) error {
  1474  	cmd := exec.Command(
  1475  		"gcloud", "projects", "add-iam-policy-binding", projectID,
  1476  		"--member", member,
  1477  		"--role", role,
  1478  	)
  1479  	return utils.Execute(cmd)
  1480  }
  1481  
  1482  func createCredentialSecret(serviceAccEmail, projectID, secretName string, k *kubectl) error {
  1483  	cmd := exec.Command(
  1484  		"gcloud", "iam", "service-accounts", "keys", "create",
  1485  		"--iam-account", serviceAccEmail,
  1486  		"--project", projectID,
  1487  		"./key.json",
  1488  	)
  1489  	if err := utils.Execute(cmd); err != nil {
  1490  		return fmt.Errorf("error creating a service account key: %v", err)
  1491  	}
  1492  	if _, err := k.exec("create", "", "ns", "cnrm-system"); err != nil {
  1493  		return fmt.Errorf("error creating cnrm-system namespace: %v", err)
  1494  	}
  1495  	if _, err := k.exec("create", "", "secret", "generic", secretName, "--from-file", "./key.json", "--namespace", "cnrm-system"); err != nil {
  1496  		return fmt.Errorf("error creating a secret containing service account key: %v", err)
  1497  	}
  1498  	rm := exec.Command("rm", "./key.json")
  1499  	if err := utils.Execute(rm); err != nil {
  1500  		return fmt.Errorf("error removing the service account key: %v", err)
  1501  	}
  1502  	return nil
  1503  }
  1504  
  1505  func addIAMBindingForServiceAcc(serviceAccEmail, member, role, projectID string) error {
  1506  	addIAMBindingFunc := func() error {
  1507  		cmd := exec.Command(
  1508  			"gcloud", "iam", "service-accounts", "add-iam-policy-binding", serviceAccEmail,
  1509  			"--member", member,
  1510  			"--role", role,
  1511  			"--project", projectID,
  1512  		)
  1513  		return utils.Execute(cmd)
  1514  	}
  1515  	return backoff.Retry(addIAMBindingFunc, backoff.NewExponentialBackOff())
  1516  }
  1517  
  1518  func createGKECluster(containerClient *containerBeta.Service, clusterName, projectID, location string, log logr.Logger) error {
  1519  	cluster := &containerBeta.Cluster{
  1520  		Name: clusterName,
  1521  		WorkloadIdentityConfig: &containerBeta.WorkloadIdentityConfig{
  1522  			IdentityNamespace: projectID + ".svc.id.goog",
  1523  		},
  1524  		InitialNodeCount: 6,
  1525  	}
  1526  	req := &containerBeta.CreateClusterRequest{
  1527  		Cluster: cluster,
  1528  	}
  1529  	parent := fmt.Sprintf("projects/%s/locations/%s", projectID, location)
  1530  	op, err := containerClient.Projects.Locations.Clusters.Create(parent, req).Do()
  1531  	if err != nil {
  1532  		return err
  1533  	}
  1534  	// Wait for cluster creation operation to finish
  1535  	err = wait.PollImmediate(10*time.Second, 10*time.Minute, func() (done bool, err error) {
  1536  		name := containerOpFullName(projectID, location, op.Name)
  1537  		res, err := containerClient.Projects.Locations.Operations.Get(name).Do()
  1538  		if err != nil {
  1539  			return false, err
  1540  		}
  1541  		if res.Status == "DONE" {
  1542  			if res.StatusMessage != "" {
  1543  				return true, fmt.Errorf("cluster creation operation failed: %v", res.StatusMessage)
  1544  			}
  1545  			return true, nil
  1546  		}
  1547  		log.Info("Waiting for cluster creation operation to finish...")
  1548  		return false, nil
  1549  	})
  1550  	if err != nil {
  1551  		return err
  1552  	}
  1553  	// Wait for cluster to be in a RUNNING state
  1554  	err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (done bool, err error) {
  1555  		name := clusterFullName(projectID, location, clusterName)
  1556  		cluster, err := containerClient.Projects.Locations.Clusters.Get(name).Do()
  1557  		if err != nil {
  1558  			return false, err
  1559  		}
  1560  		if cluster.Status == "RUNNING" {
  1561  			return true, nil
  1562  		}
  1563  		log.Info("Waiting for cluster to be in RUNNING state...", "currentState", cluster.Status)
  1564  		return false, nil
  1565  	})
  1566  	if err != nil {
  1567  		return err
  1568  	}
  1569  	return nil
  1570  }
  1571  
  1572  func deleteGKECluster(containerClient *containerBeta.Service, clusterName, projectID, location string) error {
  1573  	name := clusterFullName(projectID, location, clusterName)
  1574  	_, err := containerClient.Projects.Locations.Clusters.Delete(name).Do()
  1575  	if err != nil {
  1576  		return err
  1577  	}
  1578  	return nil
  1579  }
  1580  
  1581  func containerOpFullName(projectID, location, opName string) string {
  1582  	return fmt.Sprintf("projects/%s/locations/%s/operations/%s", projectID, location, opName)
  1583  }
  1584  
  1585  func clusterFullName(projectID, location, clusterName string) string {
  1586  	return fmt.Sprintf("projects/%s/locations/%s/clusters/%s", projectID, location, clusterName)
  1587  }
  1588  
  1589  func getKubeconfigForGKECluster(projectID, location, clusterName, outputFilePath string) error {
  1590  	cmd := exec.Command(
  1591  		"gcloud", "container", "clusters", "get-credentials", clusterName,
  1592  		"--zone", location,
  1593  		"--project", projectID,
  1594  	)
  1595  	// Override the file onto which the retrieved GKE credentials are to be
  1596  	// written in. Note that this overrides the KUBECONFIG env var even if it
  1597  	// has already been set.
  1598  	envVarOverride := "KUBECONFIG=" + outputFilePath
  1599  	cmd.Env = append(os.Environ(), envVarOverride)
  1600  	return utils.Execute(cmd)
  1601  }
  1602  
  1603  func (k *kubectl) applyStdin(stdin string, args ...string) (string, error) {
  1604  	return k.exec("apply", stdin, args...)
  1605  }
  1606  
  1607  func (k *kubectl) apply(args ...string) (stdout string, err error) {
  1608  	return k.applyStdin("", args...)
  1609  }
  1610  
  1611  func (k *kubectl) delete(args ...string) (stdout string, err error) {
  1612  	timeout := fmt.Sprintf("%vs", k.deleteTimeout.Seconds())
  1613  	args = append(args, "--timeout", timeout)
  1614  	stdout, err = k.exec("delete", "", args...)
  1615  	if err != nil && strings.Contains(err.Error(), "Error from server (NotFound)") {
  1616  		// The resource is already gone.
  1617  		return "", nil
  1618  	}
  1619  	return stdout, err
  1620  }
  1621  
  1622  func (k *kubectl) get(args ...string) (stdout string, err error) {
  1623  	return k.exec("get", "", args...)
  1624  }
  1625  
  1626  func (k *kubectl) exec(command, stdin string, args ...string) (stdout string, err error) {
  1627  	if k.kubeconfigPath == "" {
  1628  		return "", fmt.Errorf("attempted to execute a kubectl command without a specified kubeconfig")
  1629  	}
  1630  	args = append([]string{command}, args...)
  1631  	args = append(args, "--kubeconfig", k.kubeconfigPath)
  1632  	cmd := exec.Command("kubectl", args...)
  1633  	if stdin != "" {
  1634  		cmd.Stdin = bytes.NewBufferString(stdin)
  1635  	}
  1636  	return utils.ExecuteAndCaptureOutput(cmd)
  1637  }
  1638  
  1639  func downloadAndExtractOperatorReleaseTarball(version, outputDir string) error {
  1640  	tarballGCSPath := fmt.Sprintf("gs://%v/%v/%v", OPERATOR_RELEASE_BUCKET, version, OPERATOR_RELEASE_TARBALL)
  1641  	return utils.DownloadAndExtractTarballAt(tarballGCSPath, outputDir)
  1642  }
  1643  
  1644  func downloadAndExtractKCCReleaseTarball(version, outputDir string) error {
  1645  	tarballGCSPath := fmt.Sprintf("gs://%v/%v/%v", KCC_RELEASE_BUCKET, version, KCC_RELEASE_TARBALL)
  1646  	return utils.DownloadAndExtractTarballAt(tarballGCSPath, outputDir)
  1647  }
  1648  
  1649  func createTempDir(namePrefix string) (path string, err error) {
  1650  	// Creates a directory in /tmp whose name starts with
  1651  	// the given namePrefix.
  1652  	return ioutil.TempDir("", namePrefix)
  1653  }
  1654  
  1655  func writeToFile(content string, filePath string) error {
  1656  	fileMode := os.FileMode(0644) // -rw-r--r--
  1657  	return ioutil.WriteFile(filePath, []byte(content), fileMode)
  1658  }
  1659  
  1660  func (c *cluster) createNamespace(namespace string) error {
  1661  	ns := &v1.Namespace{
  1662  		ObjectMeta: metav1.ObjectMeta{
  1663  			Name: namespace,
  1664  		},
  1665  	}
  1666  	_, err := c.clientset.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{})
  1667  	return err
  1668  }
  1669  
  1670  func (c *cluster) deleteNamespace(namespace string) error {
  1671  	return c.clientset.CoreV1().Namespaces().Delete(context.Background(), namespace, metav1.DeleteOptions{})
  1672  }
  1673  
  1674  func (c *cluster) addProjectIDAnnotationToNamespace(namespace, projectID string) error {
  1675  	getFunc := func() (interface{}, error) {
  1676  		return c.getNamespace(namespace)
  1677  	}
  1678  	res, err := c.retry(getFunc, defaultBackOff)
  1679  	if err != nil {
  1680  		return err
  1681  	}
  1682  	ns := res.(*v1.Namespace)
  1683  	annotations := getAnnotationsForNS(ns)
  1684  	annotations[k8s.ProjectIdAnnotation] = projectID
  1685  	ns.SetAnnotations(annotations)
  1686  	updateFunc := func() (interface{}, error) {
  1687  		ns, err = c.clientset.CoreV1().Namespaces().Update(context.Background(), ns, metav1.UpdateOptions{})
  1688  		if err != nil {
  1689  			return nil, fmt.Errorf("error updating namespace '%v': %v", namespace, err)
  1690  		}
  1691  		return ns, nil
  1692  	}
  1693  	_, err = c.retry(updateFunc, defaultBackOff)
  1694  	return err
  1695  }
  1696  
  1697  // retry is a helper function to retry a function with the given backoff policy.
  1698  // It will return the original error from the input function if it still fails after retries.
  1699  //
  1700  // Note that we have frequently observed transient connection lost issues in operator e2e test,
  1701  // the mitigation is to retry on almost all the requests to the k8s API server. Use a backoff policy with
  1702  // a long interval if the observed transient connection issue seemingly takes a long time to recover.
  1703  func (c *cluster) retry(f func() (interface{}, error), backoff wait.Backoff) (interface{}, error) {
  1704  	var funcError error
  1705  	var res interface{}
  1706  	if err := wait.ExponentialBackoff(backoff, func() (bool, error) {
  1707  		res, funcError = f()
  1708  		if funcError != nil {
  1709  			c.log.Info("Retrying after encountering error", "error", funcError)
  1710  			return false, nil
  1711  		}
  1712  		return true, nil
  1713  	}); err != nil {
  1714  		return nil, funcError
  1715  	}
  1716  	return res, nil
  1717  }
  1718  
  1719  func (c *cluster) getKCCVersion() (string, error) {
  1720  	ns, err := c.getNamespace(k8s.CNRMSystemNamespace)
  1721  	if err != nil {
  1722  		return "", err
  1723  	}
  1724  	annotations := getAnnotationsForNS(ns)
  1725  	version, ok := annotations[k8s.VersionAnnotation]
  1726  	if !ok {
  1727  		return "", fmt.Errorf("KCC version annotation ('%v') not found in namespace '%v'", k8s.VersionAnnotation, k8s.CNRMSystemNamespace)
  1728  	}
  1729  	return version, nil
  1730  }
  1731  
  1732  func (c *cluster) getOperatorVersion() (string, error) {
  1733  	ns, err := c.getNamespace(k8s.OperatorSystemNamespace)
  1734  	if err != nil {
  1735  		return "", err
  1736  	}
  1737  	annotations := getAnnotationsForNS(ns)
  1738  	version, ok := annotations[k8s.OperatorVersionAnnotation]
  1739  	if !ok {
  1740  		return "", fmt.Errorf("KCC operator version annotation ('%v') not found in namespace '%v'", k8s.OperatorVersionAnnotation, k8s.OperatorSystemNamespace)
  1741  	}
  1742  	return version, nil
  1743  }
  1744  
  1745  func (c *cluster) getNamespace(namespace string) (*v1.Namespace, error) {
  1746  	return c.clientset.CoreV1().Namespaces().Get(context.Background(), namespace, metav1.GetOptions{})
  1747  }
  1748  
  1749  func getAnnotationsForNS(ns *v1.Namespace) map[string]string {
  1750  	annotations := ns.GetAnnotations()
  1751  	if annotations == nil {
  1752  		return make(map[string]string)
  1753  	}
  1754  	return annotations
  1755  }
  1756  
  1757  func newUniqueTestId() string {
  1758  	return randomid.New().String()
  1759  }
  1760  
  1761  func newLogger(name string) (logr.Logger, error) {
  1762  	zapConfig := zap.NewDevelopmentConfig()
  1763  	zapConfig.DisableCaller = true
  1764  	zapLog, err := zapConfig.Build()
  1765  	if err != nil {
  1766  		return logr.Discard(), err
  1767  	}
  1768  	log := zapr.NewLogger(zapLog)
  1769  	return log.WithName(name), nil
  1770  }
  1771  

View as plain text