1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package e2e
16
17 import (
18 "bytes"
19 "context"
20 "flag"
21 "fmt"
22 "io/ioutil"
23 "os"
24 "os/exec"
25 "path"
26 "strings"
27 "testing"
28 "time"
29
30 "github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/k8s"
31 "github.com/GoogleCloudPlatform/k8s-config-connector/operator/scripts/utils"
32 kcck8s "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/k8s"
33 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/randomid"
34 testgcp "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/gcp"
35
36 "github.com/blang/semver"
37 "github.com/cenkalti/backoff"
38 "github.com/go-logr/logr"
39 "github.com/go-logr/zapr"
40 "go.uber.org/zap"
41 "google.golang.org/api/cloudbilling/v1"
42 "google.golang.org/api/cloudresourcemanager/v1"
43 containerBeta "google.golang.org/api/container/v1beta1"
44 "google.golang.org/api/iam/v1"
45 v1 "k8s.io/api/core/v1"
46 "k8s.io/apimachinery/pkg/api/errors"
47 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
48 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
49 "k8s.io/apimachinery/pkg/util/wait"
50 "k8s.io/client-go/kubernetes"
51 _ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
52 "k8s.io/client-go/tools/clientcmd"
53 )
54
55 const (
56 SERVICE_ACC_ID = "cnrm-system"
57 SECRET_NAME = "gsa-key"
58
59
60 GKE_CLUSTER_ZONE1 = "us-central1-a"
61 GKE_CLUSTER_ZONE2 = "us-west1-a"
62 GKE_CLUSTER_ZONE3 = "us-west2-a"
63 KUBECTL_DELETE_TIMEOUT = 5 * time.Minute
64
65 OPERATOR_RELEASE_BUCKET = "kcc-operator-internal"
66 OPERATOR_RELEASE_TARBALL = "release-bundle.tar.gz"
67 KCC_RELEASE_BUCKET = "cnrm"
68 KCC_RELEASE_TARBALL = "release-bundle.tar.gz"
69
70
71
72
73 BASE_VERSION_SHA = "4119846"
74 )
75
76 var (
77 SERVICES = []string{
78 "container.googleapis.com",
79 "iamcredentials.googleapis.com",
80 "artifactregistry.googleapis.com",
81 }
82 organization = testgcp.GetOrgID(nil)
83 billingAccount = testgcp.GetBillingAccountID(nil)
84 f = &flags{}
85 defaultBackOff = wait.Backoff{Steps: 5, Duration: 500 * time.Millisecond, Factor: 1.5}
86 longIntervalBackOff = wait.Backoff{Steps: 3, Duration: 2 * time.Minute, Factor: 1}
87 )
88
89 type TestOptions struct {
90 OrganizationID string
91 BillingAccountID string
92 ServiceAccountID string
93 GKEClusterLocation string
94 BaseVersionSHA string
95 ProjectID string
96 SecretName string
97 }
98
99 type flags struct {
100 projectID string
101 cleanup bool
102 version string
103 }
104
105 type cluster struct {
106 kubectl *kubectl
107 clientset *kubernetes.Clientset
108 log logr.Logger
109 }
110
111 type kubectl struct {
112 kubeconfigPath string
113 deleteTimeout time.Duration
114 }
115
116 type configConnectorSample struct {
117 configConnectorClusterModeWorkloadIdentityYAMLPath string
118 configConnectorClusterModeGCPIdentityYAMLPath string
119 configConnectorNamespacedModeYAMLPath string
120 configConnectorContextYAMLPath string
121 }
122
123 type cleanupFunc func() error
124
125 func TestMain(m *testing.M) {
126 flag.StringVar(&f.projectID, "project-id", "", "Project ID that will be used for the project created for E2E tests.")
127 flag.BoolVar(&f.cleanup, "cleanup", true, "If true, project and clusters created for testing will be deleted before exiting the test suite. "+
128 "Set to false if you want to keep clusters for debugging when running the test locally.")
129 flag.StringVar(&f.version, "version", "latest", "Version of the KCC Operator to use for E2E tests. "+
130 "The version of the KCC Operator is defined by SHORT_SHA in release.sh. Use the corresponding SHORT_SHA value to run the e2e test against a particular commit. "+
131 "The default value is 'latest', which represents the version of the last green canary candidate promoted by the periodic-kcc-operator-release prow job.")
132 flag.Parse()
133 if f.projectID == "" {
134 fmt.Println("error parsing command line flags: project-id is required")
135 os.Exit(1)
136 }
137
138 log, err := newLogger("TestMain")
139 if err != nil {
140 fmt.Printf("error creating logger: %v", err)
141 os.Exit(1)
142 }
143
144 log.Info("Setting up a project for E2E tests...")
145 deleteProject, err := setupProject(organization, f.projectID, billingAccount, SERVICE_ACC_ID, log)
146 if err != nil {
147 log.Error(err, "error setting up project\r\n",
148 "Organization", organization, "ProjectID", f.projectID,
149 "BillingAccount", billingAccount, "ServiceID", SERVICE_ACC_ID)
150 cleanUpProject(deleteProject, f.cleanup, log)
151 os.Exit(1)
152 }
153 log.Info("Beginning tests...")
154 exitCode := m.Run()
155 cleanUpProject(deleteProject, f.cleanup, log)
156 os.Exit(exitCode)
157 }
158
159 func TestKCCInstallAndUninstall_Namespaced(t *testing.T) {
160 t.Parallel()
161 testOptions := newTestOptions()
162 testOptions.GKEClusterLocation = GKE_CLUSTER_ZONE1
163 testId, log, cluster, teardown := setup(t, testOptions)
164 if f.cleanup {
165 defer teardown()
166 }
167
168 manifestsDir, sample, err := getOperatorReleaseAssetsForVersion(f.version, testOptions.ServiceAccountID, testOptions.ProjectID, log)
169 if err != nil {
170 t.Fatalf("error getting operator release assets for version '%v': %v", f.version, err)
171 }
172 log.Info("Installing the operator...")
173 if err := cluster.installOperator(manifestsDir); err != nil {
174 t.Fatalf("error installing the operator: %v", err)
175 }
176 log.Info("Installing KCC...")
177 if err := cluster.installKCC(sample.configConnectorNamespacedModeYAMLPath); err != nil {
178 t.Fatalf("error installing KCC: %v", err)
179 }
180 namespace := "e2e-test-namespace"
181 if err := cluster.createNamespace(namespace); err != nil {
182 t.Fatalf("error creating namespace '%v': %v", namespace, err)
183 }
184 if err := cluster.enableKCCForNamespace(namespace, sample.configConnectorContextYAMLPath, testOptions.ServiceAccountID, testOptions.ProjectID); err != nil {
185 t.Fatalf("error enabling KCC for namespace '%v': %v", namespace, err)
186 }
187 if err := cluster.addProjectIDAnnotationToNamespace(namespace, f.projectID); err != nil {
188 t.Fatalf("error annotating namespace '%v' with the project ID: %v", namespace, err)
189 }
190 kccVersion, err := cluster.getKCCVersion()
191 if err != nil {
192 t.Fatalf("error determining KCC version: %v", err)
193 }
194 log.Info("Downloading and extracting KCC release tarball ...", "version", kccVersion)
195 kccReleaseAssetsDir, err := createTempDir("e2e-kcc-release-assets")
196 if err != nil {
197 t.Fatalf("error creating temporary directory for KCC release assets: %v", err)
198 }
199 if err := downloadAndExtractKCCReleaseTarball(kccVersion, kccReleaseAssetsDir); err != nil {
200 t.Fatalf("error downloading and extracting KCC with version '%v': %v", kccVersion, err)
201 }
202 repoName, repoYAMLDir, err := getArtifactRegistryRepositorySample(kccReleaseAssetsDir, testId)
203 if err != nil {
204 t.Fatalf("error getting ArtifactRegistryRepository sample from KCC release assets: %v", err)
205 }
206 log.Info("Creating ArtifactRegistryRepository...")
207 if err := cluster.createArtifactRegistryRepository(namespace, repoName, repoYAMLDir); err != nil {
208 t.Fatalf("error creating ArtifactRegistryRepository: %v", err)
209 }
210 log.Info("Deleting ArtifactRegistryRepository...")
211 if err := cluster.deleteArtifactRegistryRepository(namespace, repoName); err != nil {
212 t.Fatal(err)
213 }
214 log.Info("Uninstalling KCC...")
215 if err := cluster.uninstallKCC(); err != nil {
216 t.Fatalf("error uninstalling KCC: %v", err)
217 }
218 }
219
220 func TestKCCInstallAnd_Delete_Namespace_In_Namespaced_Mode(t *testing.T) {
221 t.Parallel()
222 testOptions := newTestOptions()
223 testOptions.GKEClusterLocation = GKE_CLUSTER_ZONE3
224 testId, log, cluster, teardown := setup(t, testOptions)
225 if f.cleanup {
226 defer teardown()
227 }
228 manifestsDir, sample, err := getOperatorReleaseAssetsForVersion(f.version, testOptions.ServiceAccountID, testOptions.ProjectID, log)
229 if err != nil {
230 t.Fatalf("error getting operator release assets for version '%v': %v", f.version, err)
231 }
232 log.Info("Installing the operator...")
233 if err := cluster.installOperator(manifestsDir); err != nil {
234 t.Fatalf("error installing the operator: %v", err)
235 }
236 log.Info("Installing KCC...")
237 if err := cluster.installKCC(sample.configConnectorNamespacedModeYAMLPath); err != nil {
238 t.Fatalf("error installing KCC: %v", err)
239 }
240 namespace := "e2e-test-namespace"
241 if err := cluster.createNamespace(namespace); err != nil {
242 t.Fatalf("error creating namespace '%v': %v", namespace, err)
243 }
244 if err := cluster.enableKCCForNamespace(namespace, sample.configConnectorContextYAMLPath, testOptions.ServiceAccountID, testOptions.ProjectID); err != nil {
245 t.Fatalf("error enabling KCC for namespace '%v': %v", namespace, err)
246 }
247 if err := cluster.addProjectIDAnnotationToNamespace(namespace, f.projectID); err != nil {
248 t.Fatalf("error annotating namespace '%v' with the project ID: %v", namespace, err)
249 }
250 kccVersion, err := cluster.getKCCVersion()
251 if err != nil {
252 t.Fatalf("error determining KCC version: %v", err)
253 }
254 log.Info("Downloading and extracting KCC release tarball ...", "version", kccVersion)
255 kccReleaseAssetsDir, err := createTempDir("e2e-kcc-release-assets")
256 if err != nil {
257 t.Fatalf("error creating temporary directory for KCC release assets: %v", err)
258 }
259 if err := downloadAndExtractKCCReleaseTarball(kccVersion, kccReleaseAssetsDir); err != nil {
260 t.Fatalf("error downloading and extracting KCC with version '%v': %v", kccVersion, err)
261 }
262 repoName, repoYAMLDir, err := getArtifactRegistryRepositorySample(kccReleaseAssetsDir, testId)
263 if err != nil {
264 t.Fatalf("error getting ArtifactRegistryRepository sample from KCC release assets: %v", err)
265 }
266 log.Info("Creating ArtifactRegistryRepository...")
267 if err := cluster.createArtifactRegistryRepository(namespace, repoName, repoYAMLDir); err != nil {
268 t.Fatalf("error creating ArtifactRegistryRepository: %v", err)
269 }
270
271
272 log.Info("Adding custom finalizer to prevent deletion...")
273 extraFinalizer := "extra-finalizer"
274 if err := cluster.addFinalizerToArtifactRegistryRepository(namespace, repoName, extraFinalizer); err != nil {
275 t.Fatalf("error adding finalizer to ArtifactRegistryRepository: %v", err)
276 }
277 log.Info("Deleting Namespace...")
278 if err := cluster.deleteNamespace(namespace); err != nil {
279 t.Fatalf("error deleting namespace: %v", err)
280 }
281
282
283 log.Info("Deleting ArtifactRegistryRepository...")
284 if err := cluster.deleteArtifactRegistryRepository(namespace, repoName, "--wait=false"); err != nil {
285 t.Fatal(err)
286 }
287
288
289 log.Info("Waiting for CNRM finalizer to be removed from ArtifactRegistryRepository...")
290 if err := cluster.waitForCNRMFinalizersToBeRemovedFromArtifactRegistryRepository(namespace, repoName); err != nil {
291 t.Fatalf("error waiting for CNRM finalizer to be removed from ArtifactRegistryRepository: %v", err)
292 }
293
294 log.Info("Verifying the ConfigConnectorContext still exists but is unhealthy...")
295 if err := cluster.waitForConfigConnectorContextToBeUnhealthy(namespace, k8s.ConfigConnectorContextAllowedName); err != nil {
296 t.Fatalf("error verifying the ConfigConnectorContext's health: %v", err)
297 }
298 log.Info("Removing custom finalizer to enable deletion...")
299 if err := cluster.removeFinalizerToArtifactRegistryRepository(namespace, repoName, extraFinalizer); err != nil {
300 t.Fatalf("error removing finalizer from ArtifactRegistryRepository: %v", err)
301 }
302 log.Info("Waiting for ConfigConnectorContext to be removed...")
303 if err := cluster.waitForConfigConnectorContextToBeRemoved(namespace, k8s.ConfigConnectorContextAllowedName); err != nil {
304 t.Fatalf("error waiting for ConfigConnectorContextToBeRemoved: %v", err)
305 }
306 log.Info("Waiting for namespace to be deleted...")
307 if err := cluster.waitForNamespaceToBeDeleted(namespace); err != nil {
308 t.Fatalf("error waiting for namespace to be deleted")
309 }
310 }
311
312 func TestKCCInstallAndUninstall_Cluster_WorkloadIdentity(t *testing.T) {
313 t.Parallel()
314 testOptions := newTestOptions()
315 testOptions.GKEClusterLocation = GKE_CLUSTER_ZONE2
316 testId, log, cluster, teardown := setup(t, testOptions)
317 if f.cleanup {
318 defer teardown()
319 }
320
321 manifestsDir, sample, err := getOperatorReleaseAssetsForVersion(f.version, testOptions.ServiceAccountID, testOptions.ProjectID, log)
322 if err != nil {
323 t.Fatalf("error getting operator release assets for version '%v': %v", f.version, err)
324 }
325 log.Info("Installing the operator...")
326 if err := cluster.installOperator(manifestsDir); err != nil {
327 t.Fatalf("error installing the operator: %v", err)
328 }
329 log.Info("Installing KCC in cluster mode with workload identity...")
330 if err := cluster.installKCC(sample.configConnectorClusterModeWorkloadIdentityYAMLPath); err != nil {
331 t.Fatalf("error installing KCC: %v", err)
332 }
333 namespace := "e2e-test-namespace"
334 if err := cluster.createNamespace(namespace); err != nil {
335 t.Fatalf("error creating namespace '%v': %v", namespace, err)
336 }
337 if err := cluster.addProjectIDAnnotationToNamespace(namespace, f.projectID); err != nil {
338 t.Fatalf("error annotating namespace '%v' with the project ID: %v", namespace, err)
339 }
340 kccVersion, err := cluster.getKCCVersion()
341 if err != nil {
342 t.Fatalf("error determining KCC version: %v", err)
343 }
344 log.Info("Downloading and extracting KCC release tarball ...", "version", kccVersion)
345 kccReleaseAssetsDir, err := createTempDir("e2e-kcc-release-assets")
346 if err != nil {
347 t.Fatalf("error creating temporary directory for KCC release assets: %v", err)
348 }
349 if err := downloadAndExtractKCCReleaseTarball(kccVersion, kccReleaseAssetsDir); err != nil {
350 t.Fatalf("error downloading and extracting KCC with version '%v': %v", kccVersion, err)
351 }
352 repoName, repoYAMLDir, err := getArtifactRegistryRepositorySample(kccReleaseAssetsDir, testId)
353 if err != nil {
354 t.Fatalf("error getting ArtifactRegistryRepository sample from KCC release assets: %v", err)
355 }
356 log.Info("Creating ArtifactRegistryRepository...")
357 if err := cluster.createArtifactRegistryRepository(namespace, repoName, repoYAMLDir); err != nil {
358 t.Fatalf("error creating ArtifactRegistryRepository: %v", err)
359 }
360 log.Info("Deleting ArtifactRegistryRepository...")
361 if err := cluster.deleteArtifactRegistryRepository(namespace, repoName); err != nil {
362 t.Fatal(err)
363 }
364 log.Info("Uninstalling KCC...")
365 if err := cluster.uninstallKCC(); err != nil {
366 t.Fatalf("error uninstalling KCC: %v", err)
367 }
368 }
369
370 func TestKCCInstallAndUninstall_Cluster_GCPIdentity(t *testing.T) {
371 t.Parallel()
372 testOptions := newTestOptions()
373 testOptions.GKEClusterLocation = GKE_CLUSTER_ZONE2
374 testOptions.SecretName = SECRET_NAME
375 testId, log, cluster, teardown := setup(t, testOptions)
376 if f.cleanup {
377 defer teardown()
378 }
379
380 manifestsDir, sample, err := getOperatorReleaseAssetsForVersion(f.version, testOptions.ServiceAccountID, testOptions.ProjectID, log)
381 if err != nil {
382 t.Fatalf("error getting operator release assets for version '%v': %v", f.version, err)
383 }
384 log.Info("Installing the operator...")
385 if err := cluster.installOperator(manifestsDir); err != nil {
386 t.Fatalf("error installing the operator: %v", err)
387 }
388 log.Info("Installing KCC in cluster mode with GCP identity...")
389 if err := cluster.installKCC(sample.configConnectorClusterModeGCPIdentityYAMLPath); err != nil {
390 t.Fatalf("error installing KCC: %v", err)
391 }
392 namespace := "e2e-test-namespace"
393 if err := cluster.createNamespace(namespace); err != nil {
394 t.Fatalf("error creating namespace '%v': %v", namespace, err)
395 }
396 if err := cluster.addProjectIDAnnotationToNamespace(namespace, f.projectID); err != nil {
397 t.Fatalf("error annotating namespace '%v' with the project ID: %v", namespace, err)
398 }
399 kccVersion, err := cluster.getKCCVersion()
400 if err != nil {
401 t.Fatalf("error determining KCC version: %v", err)
402 }
403 log.Info("Downloading and extracting KCC release tarball ...", "version", kccVersion)
404 kccReleaseAssetsDir, err := createTempDir("e2e-kcc-release-assets")
405 if err != nil {
406 t.Fatalf("error creating temporary directory for KCC release assets: %v", err)
407 }
408 if err := downloadAndExtractKCCReleaseTarball(kccVersion, kccReleaseAssetsDir); err != nil {
409 t.Fatalf("error downloading and extracting KCC with version '%v': %v", kccVersion, err)
410 }
411 repoName, repoYAMLDir, err := getArtifactRegistryRepositorySample(kccReleaseAssetsDir, testId)
412 if err != nil {
413 t.Fatalf("error getting ArtifactRegistryRepository sample from KCC release assets: %v", err)
414 }
415 log.Info("Creating ArtifactRegistryRepository...")
416 if err := cluster.createArtifactRegistryRepository(namespace, repoName, repoYAMLDir); err != nil {
417 t.Fatalf("error creating ArtifactRegistryRepository: %v", err)
418 }
419 log.Info("Deleting ArtifactRegistryRepository...")
420 if err := cluster.deleteArtifactRegistryRepository(namespace, repoName); err != nil {
421 t.Fatal(err)
422 }
423 log.Info("Uninstalling KCC...")
424 if err := cluster.uninstallKCC(); err != nil {
425 t.Fatalf("error uninstalling KCC: %v", err)
426 }
427 }
428
429 func TestKCCInstallAndUninstallWithoutDeletingKCCResources(t *testing.T) {
430 t.Parallel()
431 testOptions := newTestOptions()
432 testOptions.GKEClusterLocation = GKE_CLUSTER_ZONE1
433 testId, log, cluster, teardown := setup(t, testOptions)
434 if f.cleanup {
435 defer teardown()
436 }
437
438 manifestsDir, sample, err := getOperatorReleaseAssetsForVersion(f.version, testOptions.ServiceAccountID, testOptions.ProjectID, log)
439 if err != nil {
440 t.Fatalf("error getting operator release assets for version '%v': %v", f.version, err)
441 }
442 log.Info("Installing the operator...")
443 if err := cluster.installOperator(manifestsDir); err != nil {
444 t.Fatalf("error installing the operator: %v", err)
445 }
446 log.Info("Installing KCC...")
447 if err := cluster.installKCC(sample.configConnectorNamespacedModeYAMLPath); err != nil {
448 t.Fatalf("error installing KCC: %v", err)
449 }
450 namespace := "e2e-test-namespace"
451 if err := cluster.createNamespace(namespace); err != nil {
452 t.Fatalf("error creating namespace '%v': %v", namespace, err)
453 }
454 if err := cluster.enableKCCForNamespace(namespace, sample.configConnectorContextYAMLPath, testOptions.ServiceAccountID, testOptions.ProjectID); err != nil {
455 t.Fatalf("error enabling KCC for namespace '%v': %v", namespace, err)
456 }
457 if err := cluster.addProjectIDAnnotationToNamespace(namespace, f.projectID); err != nil {
458 t.Fatalf("error annotating namespace '%v' with the project ID: %v", namespace, err)
459 }
460 kccVersion, err := cluster.getKCCVersion()
461 if err != nil {
462 t.Fatalf("error determining KCC version: %v", err)
463 }
464 log.Info("Downloading and extracting KCC release tarball...", "version", kccVersion)
465 kccReleaseAssetsDir, err := createTempDir("e2e-kcc-release-assets")
466 if err != nil {
467 t.Fatalf("error creating temporary directory for KCC release assets: %v", err)
468 }
469 if err := downloadAndExtractKCCReleaseTarball(kccVersion, kccReleaseAssetsDir); err != nil {
470 t.Fatalf("error downloading and extracting KCC with version '%v': %v", kccVersion, err)
471 }
472 repoName, repoYAMLDir, err := getArtifactRegistryRepositorySample(kccReleaseAssetsDir, testId)
473 if err != nil {
474 t.Fatalf("error getting ArtifactRegistryRepository sample from KCC release assets: %v", err)
475 }
476 log.Info("Creating ArtifactRegistryRepository...")
477 if err := cluster.createArtifactRegistryRepository(namespace, repoName, repoYAMLDir); err != nil {
478 t.Fatalf("error creating ArtifactRegistryRepository: %v", err)
479 }
480 log.Info("Uninstalling KCC...")
481 if err := cluster.uninstallKCC(); err != nil {
482 t.Fatalf("error uninstalling KCC: %v", err)
483 }
484 if err := checkArtifactRegistryRepositoryExistsOnGCP(repoName, f.projectID); err != nil {
485 t.Fatal(err)
486 }
487 }
488
489 func TestShouldNotBeAbleToCreateKCCResourcesIfKCCNotEnabledForNamespace(t *testing.T) {
490 t.Parallel()
491 testOptions := newTestOptions()
492 testOptions.GKEClusterLocation = GKE_CLUSTER_ZONE1
493 testId, log, cluster, teardown := setup(t, testOptions)
494 if f.cleanup {
495 defer teardown()
496 }
497
498 manifestsDir, sample, err := getOperatorReleaseAssetsForVersion(f.version, testOptions.ServiceAccountID, testOptions.ProjectID, log)
499 if err != nil {
500 t.Fatalf("error getting operator release assets for version '%v': %v", f.version, err)
501 }
502 log.Info("Installing the operator...")
503 if err := cluster.installOperator(manifestsDir); err != nil {
504 t.Fatalf("error installing the operator: %v", err)
505 }
506 log.Info("Installing KCC...")
507 if err := cluster.installKCC(sample.configConnectorNamespacedModeYAMLPath); err != nil {
508 t.Fatalf("error installing KCC: %v", err)
509 }
510 namespace := "e2e-test-namespace"
511 if err := cluster.createNamespace(namespace); err != nil {
512 t.Fatalf("error creating namespace '%v': %v", namespace, err)
513 }
514 if err := cluster.addProjectIDAnnotationToNamespace(namespace, f.projectID); err != nil {
515 t.Fatalf("error annotating namespace '%v' with the project ID: %v", namespace, err)
516 }
517 kccVersion, err := cluster.getKCCVersion()
518 if err != nil {
519 t.Fatalf("error determining KCC version: %v", err)
520 }
521 log.Info("Downloading and extracting KCC release tarball ...", "version", kccVersion)
522 kccReleaseAssetsDir, err := createTempDir("e2e-kcc-release-assets")
523 if err != nil {
524 t.Fatalf("error creating temporary directory for KCC release assets: %v", err)
525 }
526 if err := downloadAndExtractKCCReleaseTarball(kccVersion, kccReleaseAssetsDir); err != nil {
527 t.Fatalf("error downloading and extracting KCC with version '%v': %v", kccVersion, err)
528 }
529 repoName, repoYAMLDir, err := getArtifactRegistryRepositorySample(kccReleaseAssetsDir, testId)
530 if err != nil {
531 t.Fatalf("error getting ArtifactRegistryRepository sample from KCC release assets: %v", err)
532 }
533 log.Info("Creating ArtifactRegistryRepository...")
534 if err := cluster.createArtifactRegistryRepositoryShouldFail(namespace, repoName, repoYAMLDir); err != nil {
535 t.Fatalf("error creating ArtifactRegistryRepository: %v", err)
536 }
537 ok, err := cluster.doesArtifactRegistryRepositoryHaveFinalizer(namespace, repoName, k8s.KCCFinalizer)
538 if err != nil {
539 t.Fatalf("error checking if ArtifactRegistryRepository has finalizer: %v", err)
540 }
541 if ok {
542 t.Fatalf("expected ArtifactRegistryRepository to not have finalizer '%v', but it does", k8s.KCCFinalizer)
543 }
544 ok, err = cluster.doesArtifactRegistryRepositoryHaveStatusUnmanaged(namespace, repoName, repoYAMLDir)
545 if err != nil {
546 t.Fatalf("error checking if ArtifactRegistryRepository has status '%v': %v", kcck8s.Unmanaged, err)
547 }
548 if !ok {
549 t.Fatalf("expected ArtifactRegistryRepository to have status '%v', but it does not", kcck8s.Unmanaged)
550 }
551 log.Info("Deleting ArtifactRegistryRepository...")
552 if err := cluster.deleteArtifactRegistryRepository(namespace, repoName); err != nil {
553 t.Fatal(err)
554 }
555 }
556
557 func TestUpgrade(t *testing.T) {
558 t.Parallel()
559 testOptions := newTestOptions()
560 testOptions.GKEClusterLocation = GKE_CLUSTER_ZONE2
561 testId, log, cluster, teardown := setup(t, testOptions)
562 if f.cleanup {
563 defer teardown()
564 }
565
566 manifestsDir, sample, err := getOperatorReleaseAssetsForVersion(testOptions.BaseVersionSHA, testOptions.ServiceAccountID, testOptions.ProjectID, log)
567 if err != nil {
568 t.Fatalf("error getting operator release assets for version %v: %v", testOptions.BaseVersionSHA, err)
569 }
570 log.Info("Installing the base version operator...")
571 if err := cluster.installOperator(manifestsDir); err != nil {
572 t.Fatalf("error installing the base version operator: %v", err)
573 }
574 log.Info("Installing KCC...")
575 if err := cluster.installKCC(sample.configConnectorNamespacedModeYAMLPath); err != nil {
576 t.Fatalf("error installing KCC: %v", err)
577 }
578 namespace := "e2e-test-namespace"
579 if err := cluster.createNamespace(namespace); err != nil {
580 t.Fatalf("error creating namespace '%v': %v", namespace, err)
581 }
582 if err := cluster.enableKCCForNamespace(namespace, sample.configConnectorContextYAMLPath, testOptions.ServiceAccountID, testOptions.ProjectID); err != nil {
583 t.Fatalf("error enabling KCC for namespace '%v': %v", namespace, err)
584 }
585 if err := cluster.addProjectIDAnnotationToNamespace(namespace, f.projectID); err != nil {
586 t.Fatalf("error annotating namespace '%v' with the project ID: %v", namespace, err)
587 }
588 kccVersion, err := cluster.getKCCVersion()
589 if err != nil {
590 t.Fatalf("error determining KCC version: %v", err)
591 }
592 log.Info("Downloading and extracting KCC release tarball ...", "version", kccVersion)
593 kccReleaseAssetsDir, err := createTempDir("e2e-kcc-release-assets")
594 if err != nil {
595 t.Fatalf("error creating temporary directory for KCC release assets: %v", err)
596 }
597 if err := downloadAndExtractKCCReleaseTarball(kccVersion, kccReleaseAssetsDir); err != nil {
598 t.Fatalf("error downloading and extracting KCC with version '%v': %v", kccVersion, err)
599 }
600 repoName, repoYAMLDir, err := getArtifactRegistryRepositorySample(kccReleaseAssetsDir, testId)
601 if err != nil {
602 t.Fatalf("error getting ArtifactRegistryRepository sample from KCC release assets: %v", err)
603 }
604 log.Info("Creating ArtifactRegistryRepository...")
605 if err := cluster.createArtifactRegistryRepository(namespace, repoName, repoYAMLDir); err != nil {
606 t.Fatalf("error creating ArtifactRegistryRepository: %v", err)
607 }
608 log.Info("Upgrading the operator to the latest version")
609 manifestsDir, _, err = getOperatorReleaseAssetsForVersion(f.version, testOptions.ServiceAccountID, testOptions.ProjectID, log)
610 if err != nil {
611 t.Fatalf("error getting operator release assets for version '%v': %v", f.version, err)
612 }
613 log.Info("Installing the latest operator...")
614 if err := cluster.installOperator(manifestsDir); err != nil {
615 t.Fatalf("error installing the latest operator: %v", err)
616 }
617 time.Sleep(120 * time.Second)
618 if err := cluster.waitForConfigConnectorToBeHealthy(k8s.ConfigConnectorAllowedName); err != nil {
619 t.Fatalf("error waitting for ConfigConnector to be healthy: %v", err)
620 }
621 checkIfKCCHasUpgradedToTheLatestVersion(t, cluster, log)
622 log.Info("Re-applying ArtifactRegistryRepository")
623 if err := cluster.createArtifactRegistryRepository(namespace, repoName, repoYAMLDir); err != nil {
624 t.Fatalf("error re-applying ArtifactRegistryRepository: %v", err)
625 }
626 log.Info("Deleting ArtifactRegistryRepository...")
627 if err := cluster.deleteArtifactRegistryRepository(namespace, repoName); err != nil {
628 t.Fatal(err)
629 }
630 log.Info("Deleting ConfigConnectorContext...")
631 if err := cluster.deleteConfigConnectorContext(namespace, k8s.ConfigConnectorContextAllowedName); err != nil {
632 t.Fatal(err)
633 }
634 log.Info("Uninstalling KCC...")
635 if err := cluster.uninstallKCC(); err != nil {
636 t.Fatalf("error uninstalling KCC: %v", err)
637 }
638 }
639
640 func newTestOptions() TestOptions {
641 return TestOptions{
642 OrganizationID: organization,
643 BillingAccountID: billingAccount,
644 ServiceAccountID: SERVICE_ACC_ID,
645 BaseVersionSHA: BASE_VERSION_SHA,
646 ProjectID: f.projectID,
647 }
648 }
649
650 func checkIfKCCHasUpgradedToTheLatestVersion(t *testing.T, cluster *cluster, log logr.Logger) {
651 curKccVersionRaw, err := cluster.getKCCVersion()
652 if err != nil {
653 t.Fatalf("error getting the current KCC version: %v", err)
654 }
655 currentKccVersion, err := semver.ParseTolerant(curKccVersionRaw)
656 if err != nil {
657 t.Fatalf("current KCC version %v is not a valid semantic version", curKccVersionRaw)
658 }
659 latestOperatorVersionRaw, err := cluster.getOperatorVersion()
660 if err != nil {
661 t.Fatalf("error getting the latest operator version: %v", err)
662 }
663 latestOperatorVersion, err := semver.ParseTolerant(latestOperatorVersionRaw)
664 if err != nil {
665 t.Fatalf("latest Operator version %v is not a valid semantic version", curKccVersionRaw)
666 }
667 log.Info("Version checking", "currentKCCVersion", currentKccVersion, "latestOperatorVersion", latestOperatorVersion)
668
669 if currentKccVersion.Major != latestOperatorVersion.Major || currentKccVersion.Minor != latestOperatorVersion.Minor || currentKccVersion.Patch != latestOperatorVersion.Patch {
670 t.Fatalf("expect to have KCC upgraded to %v, but it's still on version %v", latestOperatorVersion, currentKccVersion)
671 }
672 }
673
674 func setup(t *testing.T, testOptions TestOptions) (testId string, log logr.Logger, cluster *cluster, teardown func()) {
675 testId = newUniqueTestId()
676 log, err := newLogger(t.Name())
677 if err != nil {
678 t.Fatalf("error creating logger: %v", err)
679 }
680 clusterName := "e2e-test-" + testId
681 cluster, cleanup, err := setupCluster(clusterName, testOptions.ProjectID, testOptions.GKEClusterLocation,
682 testOptions.ServiceAccountID, log)
683 teardown = func() {
684 if cleanup != nil {
685 log.Info("Beginning cluster cleanup...")
686 if err := cleanup(); err != nil {
687 t.Errorf("error during cluster cleanup: %v", err)
688 }
689 }
690 }
691 if err != nil {
692 teardown()
693 t.Fatalf("error setting up cluster: %v", err)
694 }
695 if err := setupIdentity(testOptions, cluster.kubectl, log); err != nil {
696 teardown()
697 t.Fatalf("error setting up cluster: %v", err)
698 }
699 return testId, log, cluster, teardown
700 }
701
702 func cleanUpProject(deleteFunc cleanupFunc, shouldCleanUp bool, log logr.Logger) {
703 if shouldCleanUp {
704 log.Info("Beginning project cleanup...")
705 if err := deleteFunc(); err != nil {
706 log.Error(err, "error during project cleanup")
707 }
708 }
709 }
710
711 func getOperatorReleaseAssetsForVersion(version, serviceAccountID, projectID string, log logr.Logger) (manifestsDir string, sample configConnectorSample, err error) {
712 log.Info("Downloading and extracting operator release tarball...", "version", version)
713 emptySample := configConnectorSample{}
714 releaseAssetsDir, err := createTempDir("e2e-operator-release-assets")
715 if err != nil {
716 return "", emptySample, fmt.Errorf("error creating temporary directory for operator release assets: %v", err)
717 }
718 if err := downloadAndExtractOperatorReleaseTarball(version, releaseAssetsDir); err != nil {
719 return "", emptySample, fmt.Errorf("error downloading and extracting operator release tarball with version '%v': %v", version, err)
720 }
721 manifestsDir = path.Join(releaseAssetsDir, "operator-system")
722 sample, err = getConfigConnectorSample(releaseAssetsDir, serviceAccountID, projectID, version)
723 if err != nil {
724 return "", emptySample, fmt.Errorf("error getting ConfigConnector sample from operator release assets: %v", err)
725 }
726 return manifestsDir, sample, nil
727 }
728
729 func (c *cluster) installOperator(operatorManifestsDir string) error {
730 if _, err := c.kubectl.apply("-f", operatorManifestsDir); err != nil {
731 return fmt.Errorf("error applying operator manifests: %v", err)
732 }
733 time.Sleep(30 * time.Second)
734 return nil
735 }
736
737 func getConfigConnectorSample(operatorReleaseAssetsDir, serviceAccountID, projectID, version string) (sample configConnectorSample, err error) {
738 emptySample := configConnectorSample{}
739 samplesDir := path.Join(operatorReleaseAssetsDir, "samples")
740 var yamlPaths []string
741 sample = configConnectorSample{
742 configConnectorClusterModeWorkloadIdentityYAMLPath: path.Join(samplesDir, "configconnector_cluster_mode_workload_identity.yaml"),
743 configConnectorClusterModeGCPIdentityYAMLPath: path.Join(samplesDir, "configconnector_cluster_mode_gcp_identity.yaml"),
744 configConnectorNamespacedModeYAMLPath: path.Join(samplesDir, "configconnector_namespaced_mode.yaml"),
745 configConnectorContextYAMLPath: path.Join(samplesDir, "configconnectorcontext_sample.yaml"),
746 }
747 yamlPaths = []string{sample.configConnectorClusterModeWorkloadIdentityYAMLPath, sample.configConnectorNamespacedModeYAMLPath, sample.configConnectorContextYAMLPath}
748 for _, yamlPath := range yamlPaths {
749 content, err := ioutil.ReadFile(yamlPath)
750 if err != nil {
751 return emptySample, fmt.Errorf("error reading YAML: %v", err)
752 }
753 s := string(content)
754
755
756 vars := map[string]string{
757 "${GSA?}": serviceAccountID,
758 "${PROJECT_ID?}": projectID,
759 "{GSA?}": serviceAccountID,
760 "{PROJECT_ID?}": projectID,
761 }
762 processOrder := []string{"${GSA?}", "${PROJECT_ID?}", "{GSA?}", "{PROJECT_ID?}"}
763 for _, k := range processOrder {
764 v := vars[k]
765 s = strings.ReplaceAll(s, k, v)
766 }
767
768
769 if err := writeToFile(s, yamlPath); err != nil {
770 return emptySample, fmt.Errorf("error updating YAML file: %v", err)
771 }
772 }
773 return sample, nil
774 }
775
776 func (c *cluster) installKCC(configConnectorYAMLPath string) error {
777 content, err := ioutil.ReadFile(configConnectorYAMLPath)
778 if err != nil {
779 return fmt.Errorf("error reading ConfigConnector YAML: %v", err)
780 }
781 c.log.Info("Applying ConfigConnector YAML", "content", string(content))
782 if _, err := c.kubectl.apply("-f", configConnectorYAMLPath); err != nil {
783 return fmt.Errorf("error applying ConfigConnector YAML: %v", err)
784 }
785 if err := c.waitForConfigConnectorToBeHealthy(k8s.ConfigConnectorAllowedName); err != nil {
786 return err
787 }
788
789
790 return c.waitForAllComponentPodsReady()
791 }
792
793 func (c *cluster) enableKCCForNamespace(namespace, configConnectorContextYAMLPath, serviceAccountId, projectID string) error {
794 c.log.Info("Setting up Workload Identity binding for namespace...", "namespace", namespace)
795 serviceAccEmail := fmt.Sprintf("%v@%v.iam.gserviceaccount.com", serviceAccountId, projectID)
796 if err := setupWorkloadIdentityForNamespace(namespace, serviceAccEmail, projectID); err != nil {
797 return fmt.Errorf("error setting up Workload Identity binding for namespace '%v': %v", namespace, err)
798 }
799
800 content, err := ioutil.ReadFile(configConnectorContextYAMLPath)
801 if err != nil {
802 return fmt.Errorf("error reading ConfigConnectorContext YAML for namespace '%v': %v", namespace, err)
803 }
804 c.log.Info("Applying ConfigConnectorContext YAML", "namespace", namespace, "content", string(content))
805 if _, err := c.kubectl.apply("-n", namespace, "-f", configConnectorContextYAMLPath); err != nil {
806 return fmt.Errorf("error applying ConfigConnectorContext YAML for namespace '%v': %v", namespace, err)
807 }
808 if err := c.waitForConfigConnectorToBeHealthy(k8s.ConfigConnectorAllowedName); err != nil {
809 return err
810 }
811 if err := c.waitForConfigConnectorContextToBeHealthy(namespace, k8s.ConfigConnectorContextAllowedName); err != nil {
812 return err
813 }
814 time.Sleep(90 * time.Second)
815 return nil
816 }
817
818 func (c *cluster) waitForConfigConnectorToBeHealthy(name string) error {
819 err := wait.PollImmediate(5*time.Second, 5*time.Minute, func() (done bool, err error) {
820 f := func() (interface{}, error) {
821 return c.kubectl.get("configconnector", name, "-o", "yaml")
822 }
823 res, err := c.retry(f, longIntervalBackOff)
824 if err != nil {
825 return false, fmt.Errorf("error getting ConfigConnector '%v': %v", name, err)
826 }
827 c.log.Info("Waiting for ConfigConnector to reach a healthy state...", "name", name)
828 return strings.Contains(res.(string), "healthy: true"), nil
829 })
830 if err != nil {
831 if err != wait.ErrWaitTimeout {
832 return err
833 }
834 out, _ := c.kubectl.get("configconnector", name, "-o", "yaml")
835 return fmt.Errorf("timed out waiting for ConfigConnector '%v' to reach a healthy state:\n%v", name, out)
836 }
837 c.log.Info("ConfigConnector has reached a healthy state", "name", name)
838 return nil
839 }
840
841 func (c *cluster) waitForAllComponentPodsReady() error {
842 c.log.Info("waiting for all component pods in 'cnrm-system' namespace to be ready...")
843 out, err := c.kubectl.exec("wait", "", "-n", "cnrm-system", "--for=condition=Ready", "pod", "--all", "--timeout=180s")
844 if err != nil {
845 return fmt.Errorf("error waiting for all component pods in 'cnrm-system' namespace to be ready: %w, output: %v", err, out)
846 }
847 return nil
848 }
849
850 func (c *cluster) waitForConfigConnectorContextToBeRemoved(namespace, name string) error {
851 err := wait.PollImmediate(5*time.Second, 5*time.Minute, func() (done bool, err error) {
852 var isDeleted bool
853 f := func() (interface{}, error) {
854 c.log.Info("Getting ConfigConnectorContext...", "namespace", namespace, "name", name)
855 out, err := c.kubectl.get("-n", namespace, "configconnectorcontext", name)
856 if err != nil {
857
858 if strings.Contains(err.Error(), "Error from server (NotFound)") {
859 isDeleted = true
860 return nil, nil
861 }
862 return nil, err
863 }
864 return out, nil
865 }
866 out, err := c.retry(f, defaultBackOff)
867 if err != nil {
868 return false, fmt.Errorf("unexpected error getting ConfigConnectorContext: %w; command output: %v", err, out)
869 }
870 if isDeleted {
871 return true, nil
872 }
873 c.log.Info("Waiting for ConfigConnectorContext to be deleted...", "namespace", namespace, "name", name, "output", out)
874 return false, nil
875 })
876 if err == nil {
877 return nil
878 }
879 if err != wait.ErrWaitTimeout {
880 return err
881 }
882 out, _ := c.kubectl.get("-n", namespace, "configconnectorcontext", name, "-o", "yaml")
883 return fmt.Errorf("timed out waiting for ConfigConnectorContext '%v/%v' to be removed:\n%v", namespace, name, out)
884 }
885
886 func (c *cluster) waitForConfigConnectorContextToBeHealthy(namespace, name string) error {
887 return c.waitForConfigConnectorContextToBeHealthyOrUnhealthy(namespace, name, true)
888 }
889
890 func (c *cluster) waitForConfigConnectorContextToBeUnhealthy(namespace, name string) error {
891 return c.waitForConfigConnectorContextToBeHealthyOrUnhealthy(namespace, name, false)
892 }
893
894 func (c *cluster) waitForConfigConnectorContextToBeHealthyOrUnhealthy(namespace, name string, healthy bool) error {
895 desiredState := "unhealthy"
896 if healthy {
897 desiredState = "healthy"
898 }
899 err := wait.PollImmediate(5*time.Second, 10*time.Minute, func() (done bool, err error) {
900 f := func() (interface{}, error) {
901 return c.kubectl.get("-n", namespace, "configconnectorcontext", name, "-o", "yaml")
902 }
903 out, err := c.retry(f, longIntervalBackOff)
904 if err != nil {
905 return false, fmt.Errorf("error getting ConfigConnectorContext '%v' for namespace '%v': %v", name, namespace, err)
906 }
907 c.log.Info(fmt.Sprintf("Waiting for ConfigConnectorContext to reach an %v state...", desiredState), "namespace", namespace, "name", name)
908 return strings.Contains(out.(string), fmt.Sprintf("healthy: %v", healthy)), nil
909 })
910 if err != nil {
911 if err != wait.ErrWaitTimeout {
912 return err
913 }
914 out, _ := c.kubectl.get("-n", namespace, "configconnectorcontext", name, "-o", "yaml")
915 return fmt.Errorf("timed out waiting for ConfigConnectorContext '%v/%v' to reach an %v state:\n%v", namespace, name, desiredState, out)
916 }
917 c.log.Info(fmt.Sprintf("ConfigConnectorContext has reached an %v state", desiredState), "namespace", namespace, "name", name)
918 return nil
919 }
920
921 func setupWorkloadIdentityForNamespace(namespace, serviceAccEmail, projectID string) error {
922 member := fmt.Sprintf("serviceAccount:%v.svc.id.goog[cnrm-system/%v%v]", projectID, k8s.ServiceAccountNamePrefix, namespace)
923 role := "roles/iam.workloadIdentityUser"
924 if err := addIAMBindingForServiceAcc(serviceAccEmail, member, role, projectID); err != nil {
925 return fmt.Errorf("error setting up Workload Identity binding: %v", err)
926 }
927 return nil
928 }
929
930 func getArtifactRegistryRepositorySample(kccReleaseAssetsDir, uniqueId string) (repoName string, repoYAMLDir string, err error) {
931 repoYAMLDir = path.Join(kccReleaseAssetsDir, "samples", "resources", "artifactregistryrepository")
932 yamlPaths, err := getYAMLFilesInDir(repoYAMLDir)
933 if err != nil {
934 return "", "", fmt.Errorf("error getting paths to YAML files in ArtifactRegistryRepository sample directory '%v': %v", repoYAMLDir, err)
935 }
936 for _, yamlPath := range yamlPaths {
937 b, err := ioutil.ReadFile(yamlPath)
938 if err != nil {
939 return "", "", fmt.Errorf("error reading file '%v': %v", yamlPath, err)
940 }
941 s := string(b)
942 s = strings.ReplaceAll(s, "sample", "sample"+uniqueId)
943 s = strings.ReplaceAll(s, "dep", "dep"+uniqueId)
944
945
946 if err := writeToFile(s, yamlPath); err != nil {
947 return "", "", fmt.Errorf("error updating file '%v': %v", yamlPath, err)
948 }
949 }
950 repoName, err = getRepoNameFromArtifactRegistryRepositorySampleDir(repoYAMLDir)
951 if err != nil {
952 return "", "", fmt.Errorf("error getting name of ArtifactRegistryRepository for ArtifactRegistryRepository sample directory '%v': %v", repoYAMLDir, err)
953 }
954 return repoName, repoYAMLDir, nil
955 }
956
957 func getRepoNameFromArtifactRegistryRepositorySampleDir(repoYAMLDir string) (string, error) {
958 unstructs := make([]*unstructured.Unstructured, 0)
959 yamlPaths, err := getYAMLFilesInDir(repoYAMLDir)
960 if err != nil {
961 return "", fmt.Errorf("error getting paths to YAML files in directory '%v': %v", repoYAMLDir, err)
962 }
963 for _, yamlPath := range yamlPaths {
964 u, err := utils.ReadFileToUnstructs(yamlPath)
965 if err != nil {
966 return "", fmt.Errorf("error converting file '%v' to unstructs: %v", yamlPath, err)
967 }
968 unstructs = append(unstructs, u...)
969 }
970 repoNames := make([]string, 0)
971 for _, u := range unstructs {
972 if u.GetKind() == "ArtifactRegistryRepository" {
973 repoNames = append(repoNames, u.GetName())
974 }
975 }
976 switch len(repoNames) {
977 case 0:
978 return "", fmt.Errorf("no ArtifactRegistryRepository found in directory '%v'", repoYAMLDir)
979 case 1:
980 return repoNames[0], nil
981 default:
982 return "", fmt.Errorf("multiple ArtifactRegistryRepositories found in directory '%v'", repoYAMLDir)
983 }
984 }
985
986 func getYAMLFilesInDir(dir string) (yamlPaths []string, err error) {
987 yamlPaths = make([]string, 0)
988 fileInfos, err := ioutil.ReadDir(dir)
989 if err != nil {
990 return []string{}, fmt.Errorf("error reading directory '%v': %v", dir, err)
991 }
992 for _, fi := range fileInfos {
993 if fi.IsDir() {
994 continue
995 }
996 if !strings.HasSuffix(fi.Name(), ".yaml") {
997 continue
998 }
999 yamlPaths = append(yamlPaths, path.Join(dir, fi.Name()))
1000 }
1001 return yamlPaths, nil
1002 }
1003
1004 func (c *cluster) createArtifactRegistryRepository(namespace, repoName, repoYAMLDir string) error {
1005 if err := c.createArtifactRegistryRepositoryAndWait(namespace, repoName, repoYAMLDir); err != nil {
1006 if err == wait.ErrWaitTimeout {
1007 out, _ := c.kubectl.get("-n", namespace, "artifactregistryrepository", repoName, "-o", "yaml")
1008 return fmt.Errorf("timed out waiting for ArtifactRegistryRepository to reach an UpToDate state:\n%v", out)
1009 }
1010 return err
1011 }
1012 return nil
1013 }
1014
1015 func (c *cluster) createArtifactRegistryRepositoryShouldFail(namespace, repoName, repoYAMLDir string) error {
1016 if err := c.createArtifactRegistryRepositoryAndWait(namespace, repoName, repoYAMLDir); err != nil {
1017 if err == wait.ErrWaitTimeout {
1018 return nil
1019 }
1020 return err
1021 }
1022
1023 out, _ := c.kubectl.get("-n", namespace, "artifactregistryrepository", repoName, "-o", "yaml")
1024 return fmt.Errorf("expected creation of ArtifactRegistryRepository to fail, but got:\n%v", out)
1025 }
1026
1027 func (c *cluster) createArtifactRegistryRepositoryAndWait(namespace, repoName, repoYAMLDir string) error {
1028 if _, err := c.kubectl.apply("-n", namespace, "-f", repoYAMLDir); err != nil {
1029 return fmt.Errorf("error applying ArtifactRegistryRepository: %v", err)
1030 }
1031 return wait.PollImmediate(5*time.Second, 2*time.Minute, func() (done bool, err error) {
1032 c.log.Info("Getting ArtifactRegistryRepository...", "name", repoName)
1033 f := func() (interface{}, error) {
1034 return c.kubectl.get("-n", namespace, "artifactregistryrepository", repoName, "-o", "yaml")
1035 }
1036
1037
1038 out, err := c.retry(f, longIntervalBackOff)
1039 if err != nil {
1040 return false, fmt.Errorf("error getting ArtifactRegistryRepository '%v/%v': %v", namespace, repoName, err)
1041 }
1042 c.log.Info("Waiting for ArtifactRegistryRepository to reach an UpToDate state...", "name", repoName)
1043 return strings.Contains(out.(string), "UpToDate"), nil
1044 })
1045 }
1046
1047 func (c *cluster) waitForCNRMFinalizersToBeRemovedFromArtifactRegistryRepository(namespace, repoName string) error {
1048 waitFunc := func() (done bool, err error) {
1049 ok, err := c.doesArtifactRegistryRepositoryHaveFinalizer(namespace, repoName, k8s.KCCFinalizer)
1050 if err != nil {
1051 return false, fmt.Errorf("error checking for the finalizer on ArtifactRegistryRepository: %w", err)
1052 }
1053 return !ok, nil
1054 }
1055 if err := wait.PollImmediate(5*time.Second, 5*time.Minute, waitFunc); err != nil {
1056 if err != wait.ErrWaitTimeout {
1057 return err
1058 }
1059 out, _ := c.kubectl.get("-n", namespace, "artifactregistryrepository", repoName, "-o", "yaml")
1060 return fmt.Errorf("timed out waiting for the CNRM finalizers to be removed from ArtifactRegistryRepository '%v/%v':\n%v", namespace, repoName, out)
1061 }
1062 return nil
1063 }
1064
1065 func (c *cluster) getArtifactRegistryRepositoryUnstructured(namespace, repoName string) (*unstructured.Unstructured, error) {
1066 out, err := c.kubectl.get("-n", namespace, "artifactregistryrepository", repoName, "-o", "yaml")
1067 if err != nil {
1068 return nil, fmt.Errorf("error getting ArtifactRegistryRepository '%v/%v': %v", namespace, repoName, err)
1069 }
1070 repoUnstruct, err := utils.BytesToUnstruct([]byte(out))
1071 if err != nil {
1072 return nil, fmt.Errorf("error converting '%v' to unstruct: %w", out, err)
1073 }
1074 return repoUnstruct, err
1075 }
1076
1077 func (c *cluster) applyUnstructured(u *unstructured.Unstructured) error {
1078 bytes, err := utils.UnstructToYaml(u)
1079 if err != nil {
1080 return fmt.Errorf("error converting unstruct to yaml: %w", err)
1081 }
1082 if _, err := c.kubectl.applyStdin(string(bytes), "-f", "-"); err != nil {
1083 return fmt.Errorf("error applying %v after adding finalizer: %w", u.GetKind(), err)
1084 }
1085 return nil
1086 }
1087
1088 func (c *cluster) removeFinalizerToArtifactRegistryRepository(namespace, repoName, finalizer string) error {
1089 repoUnstruct, err := c.getArtifactRegistryRepositoryUnstructured(namespace, repoName)
1090 if err != nil {
1091 return err
1092 }
1093 finalizers := append(repoUnstruct.GetFinalizers(), finalizer)
1094 var newFinalizers []string
1095 for _, f := range finalizers {
1096 if f == finalizer {
1097 continue
1098 }
1099 newFinalizers = append(newFinalizers, f)
1100 }
1101 repoUnstruct.SetFinalizers(newFinalizers)
1102 return c.applyUnstructured(repoUnstruct)
1103 }
1104
1105 func (c *cluster) addFinalizerToArtifactRegistryRepository(namespace, repoName, finalizer string) error {
1106 repoUnstruct, err := c.getArtifactRegistryRepositoryUnstructured(namespace, repoName)
1107 if err != nil {
1108 return err
1109 }
1110 finalizers := append(repoUnstruct.GetFinalizers(), finalizer)
1111 repoUnstruct.SetFinalizers(finalizers)
1112 return c.applyUnstructured(repoUnstruct)
1113 }
1114
1115 func (c *cluster) doesArtifactRegistryRepositoryHaveFinalizer(namespace, repoName, finalizer string) (ok bool, err error) {
1116 repoUnstruct, err := c.getArtifactRegistryRepositoryUnstructured(namespace, repoName)
1117 if err != nil {
1118 return false, err
1119 }
1120 for _, f := range repoUnstruct.GetFinalizers() {
1121 if finalizer == f {
1122 return true, nil
1123 }
1124 }
1125 return false, nil
1126 }
1127
1128 func (c *cluster) doesArtifactRegistryRepositoryHaveStatusUnmanaged(namespace, repoName, finalizer string) (ok bool, err error) {
1129 repoUnstruct, err := c.getArtifactRegistryRepositoryUnstructured(namespace, repoName)
1130 if err != nil {
1131 return false, err
1132 }
1133 r, err := kcck8s.NewResource(repoUnstruct)
1134 if err != nil {
1135 return false, err
1136 }
1137 condition, ok := kcck8s.GetReadyCondition(r)
1138 if !ok {
1139 return false, nil
1140 }
1141 return condition.Reason == kcck8s.Unmanaged, nil
1142 }
1143
1144 func (c *cluster) deleteArtifactRegistryRepository(namespace, repoName string, extraArgs ...string) error {
1145 args := []string{"-n", namespace, "artifactregistryrepository", repoName}
1146 args = append(args, extraArgs...)
1147 f := func() (interface{}, error) {
1148 return c.kubectl.delete(args...)
1149 }
1150 _, err := c.retry(f, defaultBackOff)
1151 if err != nil {
1152 return fmt.Errorf("error deleting ArtifactRegistryRepository: %v", err)
1153 }
1154 return nil
1155 }
1156
1157 func (c *cluster) deleteConfigConnectorContext(namespace, name string) error {
1158 args := []string{"-n", namespace, "configconnectorcontext", name}
1159 f := func() (interface{}, error) {
1160 if _, err := c.kubectl.delete(args...); err != nil {
1161 c.log.Info("error deleting ConfigConnectorContext...", "error", err)
1162 return nil, err
1163 }
1164 return nil, nil
1165 }
1166 if _, err := c.retry(f, longIntervalBackOff); err != nil {
1167 return fmt.Errorf("error deleting ConfigConnectorContext: %v", err)
1168 }
1169 return c.waitForConfigConnectorContextToBeRemoved(namespace, name)
1170 }
1171
1172 func (c *cluster) uninstallKCC() error {
1173 c.log.Info("deleting the ConfigConnector object")
1174 f := func() (interface{}, error) {
1175 if _, err := c.kubectl.delete("configconnector", k8s.ConfigConnectorAllowedName); err != nil {
1176 c.log.Info("error deleting ConfigConnector...", "error", err)
1177 return nil, err
1178 }
1179 return nil, nil
1180 }
1181 if _, err := c.retry(f, longIntervalBackOff); err != nil {
1182 return fmt.Errorf("error deleting ConfigConnectors: %v", err)
1183 }
1184 c.log.Info("Asserting that the ConfigConnector object is gone")
1185 out, err := c.kubectl.get("configconnector")
1186 if err != nil {
1187 return fmt.Errorf("error getting ConfigConnectors: %v", err)
1188 }
1189 if !strings.Contains(out, "No resources found") {
1190 return fmt.Errorf("expected no ConfigConnectors to exist, but got:\n%v", out)
1191 }
1192
1193
1194
1195
1196
1197
1198
1199 crdAssertionFunc := func() (interface{}, error) {
1200 c.log.Info("Asserting that resource CRDs are deleted")
1201 out, err = c.kubectl.get("crds", "--selector", "cnrm.cloud.google.com/managed-by-kcc=true")
1202 if err != nil {
1203 return nil, fmt.Errorf("error getting KCC CRDs: %v", err)
1204 }
1205 if !strings.Contains(out, "No resources found") {
1206 return nil, fmt.Errorf("expected KCC CRDs to not exist, but got:\n%v", out)
1207 }
1208 return out, nil
1209 }
1210 _, err = c.retry(crdAssertionFunc, defaultBackOff)
1211 if err != nil {
1212 return fmt.Errorf("unexpected error asserting that resource CRDs are deleted: %w", err)
1213 }
1214
1215 out, err = c.kubectl.get("validatingwebhookconfiguration")
1216 if err != nil {
1217 return fmt.Errorf("error getting ValidatingWebhookConfigurations: %v", err)
1218 }
1219 if strings.Contains(out, k8s.CNRMDomain) {
1220 return fmt.Errorf("expected KCC validating webhooks to not exist, but got:\n%v", out)
1221 }
1222 out, err = c.kubectl.get("mutatingwebhookconfiguration")
1223 if err != nil {
1224 return fmt.Errorf("error getting MutatingWebhookConfigurations: %v", err)
1225 }
1226 if strings.Contains(out, k8s.CNRMDomain) {
1227 return fmt.Errorf("expected KCC mutating webhooks to not exist, but got:\n%v", out)
1228 }
1229 c.log.Info("Asserting that `cnrm-system` namespace is deleted")
1230 return c.waitForNamespaceToBeDeleted(k8s.CNRMSystemNamespace)
1231 }
1232
1233 func (c *cluster) waitForNamespaceToBeDeleted(namespace string) error {
1234
1235 time.Sleep(5 * time.Minute)
1236
1237 err := wait.PollImmediate(20*time.Second, 10*time.Minute, func() (done bool, err error) {
1238 c.log.Info("Getting namespace...", "namespace", namespace)
1239 var isDeleted bool
1240 f := func() (interface{}, error) {
1241 ns, err := c.getNamespace(namespace)
1242 if err != nil {
1243
1244 if errors.IsNotFound(err) {
1245 isDeleted = true
1246 return nil, nil
1247 }
1248 return nil, err
1249 }
1250 return ns, nil
1251 }
1252
1253
1254 res, err := c.retry(f, longIntervalBackOff)
1255 if err != nil {
1256 return false, fmt.Errorf("error getting namespace '%v': %v", namespace, err)
1257 }
1258 if isDeleted {
1259 return true, nil
1260 }
1261 ns := res.(*v1.Namespace)
1262 c.log.Info("Waiting for namespace to be deleted...", "namespace", namespace, "status", ns.Status)
1263 return false, nil
1264 })
1265 if err != nil {
1266 if err != wait.ErrWaitTimeout {
1267 return err
1268 }
1269 return fmt.Errorf("timed out waiting for namespace '%v' to be deleted", namespace)
1270 }
1271 return nil
1272 }
1273
1274 func checkArtifactRegistryRepositoryExistsOnGCP(repoName, projectID string) error {
1275 cmd := exec.Command("gcloud", "artifacts", "repositories", "describe", repoName,
1276 "--location", "us-west1",
1277 "--project", projectID)
1278 _, err := utils.ExecuteAndCaptureOutput(cmd)
1279 if err != nil {
1280 if strings.Contains(err.Error(), "NOT_FOUND") {
1281 return fmt.Errorf("expected project '%v' to have Artifact Registry Repository named '%v', but got:\n%v", projectID, repoName, err)
1282 }
1283 return fmt.Errorf("error checking if Artifact Registry Repository exists on GCP: %v", err)
1284 }
1285 return nil
1286 }
1287
1288 func setupCluster(clusterName, projectID, location, serviceAccountId string, log logr.Logger) (*cluster, cleanupFunc, error) {
1289 var cleanup cleanupFunc
1290 log.Info("Creating a Container client...")
1291 ctx := context.Background()
1292 container, err := containerBeta.NewService(ctx)
1293 if err != nil {
1294 return nil, cleanup, fmt.Errorf("error creating Container client: %v", err)
1295 }
1296 log.Info("Creating a GKE cluster...", "name", clusterName)
1297 if err := createGKECluster(container, clusterName, projectID, location, log); err != nil {
1298 return nil, cleanup, fmt.Errorf("error creating GKE cluster with name '%v': %v", clusterName, err)
1299 }
1300 cleanup = func() error {
1301 log.Info("Deleting GKE cluster...", "name", clusterName)
1302 if err := deleteGKECluster(container, clusterName, projectID, location); err != nil {
1303 return fmt.Errorf("error deleting cluster with name '%v': %v", clusterName, err)
1304 }
1305 return nil
1306 }
1307
1308 log.Info("Getting the cluster's kubeconfig...")
1309 outDirForKubeconfig, err := createTempDir("e2e-" + clusterName + "-kubeconfig")
1310 if err != nil {
1311 return nil, cleanup, fmt.Errorf("error creating temporary directory for cluster's kubeconfig: %v", err)
1312 }
1313 outPathForKubeconfig := path.Join(outDirForKubeconfig, "kubeconfig.yaml")
1314 if err := getKubeconfigForGKECluster(projectID, location, clusterName, outPathForKubeconfig); err != nil {
1315 return nil, cleanup, fmt.Errorf("error getting cluster's kubeconfig: %v", err)
1316 }
1317 log.Info("Setting up a client-go Clientset...")
1318 config, err := clientcmd.BuildConfigFromFlags("", outPathForKubeconfig)
1319 if err != nil {
1320 return nil, cleanup, fmt.Errorf("error building REST client config from kubeconfig: %v", err)
1321 }
1322 clientset, err := kubernetes.NewForConfig(config)
1323 if err != nil {
1324 return nil, cleanup, fmt.Errorf("error creating client-go Clientset: %v", err)
1325 }
1326 cluster := &cluster{
1327 kubectl: &kubectl{
1328 kubeconfigPath: outPathForKubeconfig,
1329 deleteTimeout: KUBECTL_DELETE_TIMEOUT,
1330 },
1331 clientset: clientset,
1332 log: log,
1333 }
1334 return cluster, cleanup, nil
1335 }
1336
1337 func setupIdentity(testOptions TestOptions, k *kubectl, log logr.Logger) error {
1338 if testOptions.SecretName != "" {
1339 log.Info("Creating a secret containing service account key..")
1340 serviceAccEmail := fmt.Sprintf("%v@%v.iam.gserviceaccount.com", testOptions.ServiceAccountID, testOptions.ProjectID)
1341 if err := createCredentialSecret(serviceAccEmail, testOptions.ProjectID, testOptions.SecretName, k); err != nil {
1342 return err
1343 }
1344 return nil
1345 } else {
1346 log.Info("Setting up Workload Identity binding...")
1347 serviceAccEmail := fmt.Sprintf("%v@%v.iam.gserviceaccount.com", testOptions.ServiceAccountID, testOptions.ProjectID)
1348 member := fmt.Sprintf("serviceAccount:%v.svc.id.goog[cnrm-system/cnrm-controller-manager]", testOptions.ProjectID)
1349 role := "roles/iam.workloadIdentityUser"
1350 if err := addIAMBindingForServiceAcc(serviceAccEmail, member, role, testOptions.ProjectID); err != nil {
1351 return err
1352 }
1353 return nil
1354 }
1355 }
1356
1357 func setupProject(organizationID, projectID, billingAccountID, serviceAccountID string, log logr.Logger) (cleanupFunc, error) {
1358 var cleanup cleanupFunc
1359 log.Info("Creating GCP clients...")
1360 ctx := context.Background()
1361 resourceManagerClient, err := cloudresourcemanager.NewService(ctx)
1362 if err != nil {
1363 return nil, fmt.Errorf("error creating ResourceManager client: %v", err)
1364 }
1365 billingClient, err := cloudbilling.NewService(ctx)
1366 if err != nil {
1367 return nil, fmt.Errorf("error creating Billing client: %v", err)
1368 }
1369 iamClient, err := iam.NewService(ctx)
1370 if err != nil {
1371 return nil, fmt.Errorf("error creating IAM client: %v", err)
1372 }
1373 log.Info("Creating project...", "projectID", projectID)
1374 if err := createProject(resourceManagerClient, organizationID, projectID, log); err != nil {
1375 return cleanup, fmt.Errorf("error creating project with project ID '%v': %v", projectID, err)
1376 }
1377 cleanup = func() error {
1378 log.Info("Deleting project...", "projectID", projectID)
1379 if err := deleteProject(resourceManagerClient, projectID); err != nil {
1380 return fmt.Errorf("error deleting project with project ID '%v': %v", projectID, err)
1381 }
1382 return nil
1383 }
1384 log.Info("Linking project to billing account...", "billingAccount", billingAccountID)
1385 if err := linkProjectToBillingAccount(billingClient, projectID, billingAccountID); err != nil {
1386 return cleanup, fmt.Errorf("error linking project to billing account '%v'", billingAccountID)
1387 }
1388 log.Info("Enabling services for project...")
1389 if err := enableServicesForProject(projectID, SERVICES, log); err != nil {
1390 return cleanup, fmt.Errorf("error enabling services for project: %v", err)
1391 }
1392 log.Info("Setting up IAM service account...")
1393 if err := createServiceAccount(iamClient, serviceAccountID, projectID); err != nil {
1394 return cleanup, fmt.Errorf("error creating service account: %v", err)
1395 }
1396 serviceAccEmail := fmt.Sprintf("%v@%v.iam.gserviceaccount.com", serviceAccountID, projectID)
1397 if err := addIAMBindingForProject(projectID, "serviceAccount:"+serviceAccEmail, "roles/owner"); err != nil {
1398 return cleanup, fmt.Errorf("error granting service account project owner role: %v", err)
1399 }
1400 return cleanup, nil
1401 }
1402
1403 func createProject(resourceManagerClient *cloudresourcemanager.Service, organizationID, projectID string, log logr.Logger) error {
1404 project := &cloudresourcemanager.Project{
1405 ProjectId: projectID,
1406 Labels: map[string]string{
1407 "cnrm-test": "true",
1408 },
1409 Parent: &cloudresourcemanager.ResourceId{
1410 Type: "organization",
1411 Id: organizationID,
1412 },
1413 }
1414 op, err := resourceManagerClient.Projects.Create(project).Do()
1415 if err != nil {
1416 return err
1417 }
1418
1419 return wait.PollImmediate(5*time.Second, 5*time.Minute, func() (done bool, err error) {
1420 res, err := resourceManagerClient.Operations.Get(op.Name).Do()
1421 if err != nil {
1422 return false, err
1423 }
1424 if res.Done {
1425 if res.Error != nil {
1426 return true, fmt.Errorf("project creation operation failed: %v", res.Error)
1427 }
1428 return true, nil
1429 }
1430 log.Info("Waiting for project creation operation to finish...")
1431 return false, nil
1432 })
1433 }
1434
1435 func deleteProject(resourceManagerClient *cloudresourcemanager.Service, projectID string) error {
1436 _, err := resourceManagerClient.Projects.Delete(projectID).Do()
1437 if err != nil {
1438 return err
1439 }
1440 return nil
1441 }
1442
1443 func linkProjectToBillingAccount(billingClient *cloudbilling.APIService, projectID, billingAccount string) error {
1444 ba := &cloudbilling.ProjectBillingInfo{
1445 BillingAccountName: "billingAccounts/" + billingAccount,
1446 }
1447 _, err := billingClient.Projects.UpdateBillingInfo("projects/"+projectID, ba).Do()
1448 return err
1449 }
1450
1451 func enableServicesForProject(projectID string, services []string, log logr.Logger) error {
1452 for _, service := range services {
1453 log.Info("Enabling service...", "service", service)
1454 cmd := exec.Command("gcloud", "services", "enable", service, "--project", projectID)
1455 if err := utils.Execute(cmd); err != nil {
1456 return err
1457 }
1458 }
1459 return nil
1460 }
1461
1462 func createServiceAccount(iamClient *iam.Service, serviceAccountID, projectID string) error {
1463 req := &iam.CreateServiceAccountRequest{
1464 AccountId: serviceAccountID,
1465 }
1466 _, err := iamClient.Projects.ServiceAccounts.Create("projects/"+projectID, req).Do()
1467 if err != nil {
1468 return err
1469 }
1470 return nil
1471 }
1472
1473 func addIAMBindingForProject(projectID, member, role string) error {
1474 cmd := exec.Command(
1475 "gcloud", "projects", "add-iam-policy-binding", projectID,
1476 "--member", member,
1477 "--role", role,
1478 )
1479 return utils.Execute(cmd)
1480 }
1481
1482 func createCredentialSecret(serviceAccEmail, projectID, secretName string, k *kubectl) error {
1483 cmd := exec.Command(
1484 "gcloud", "iam", "service-accounts", "keys", "create",
1485 "--iam-account", serviceAccEmail,
1486 "--project", projectID,
1487 "./key.json",
1488 )
1489 if err := utils.Execute(cmd); err != nil {
1490 return fmt.Errorf("error creating a service account key: %v", err)
1491 }
1492 if _, err := k.exec("create", "", "ns", "cnrm-system"); err != nil {
1493 return fmt.Errorf("error creating cnrm-system namespace: %v", err)
1494 }
1495 if _, err := k.exec("create", "", "secret", "generic", secretName, "--from-file", "./key.json", "--namespace", "cnrm-system"); err != nil {
1496 return fmt.Errorf("error creating a secret containing service account key: %v", err)
1497 }
1498 rm := exec.Command("rm", "./key.json")
1499 if err := utils.Execute(rm); err != nil {
1500 return fmt.Errorf("error removing the service account key: %v", err)
1501 }
1502 return nil
1503 }
1504
1505 func addIAMBindingForServiceAcc(serviceAccEmail, member, role, projectID string) error {
1506 addIAMBindingFunc := func() error {
1507 cmd := exec.Command(
1508 "gcloud", "iam", "service-accounts", "add-iam-policy-binding", serviceAccEmail,
1509 "--member", member,
1510 "--role", role,
1511 "--project", projectID,
1512 )
1513 return utils.Execute(cmd)
1514 }
1515 return backoff.Retry(addIAMBindingFunc, backoff.NewExponentialBackOff())
1516 }
1517
1518 func createGKECluster(containerClient *containerBeta.Service, clusterName, projectID, location string, log logr.Logger) error {
1519 cluster := &containerBeta.Cluster{
1520 Name: clusterName,
1521 WorkloadIdentityConfig: &containerBeta.WorkloadIdentityConfig{
1522 IdentityNamespace: projectID + ".svc.id.goog",
1523 },
1524 InitialNodeCount: 6,
1525 }
1526 req := &containerBeta.CreateClusterRequest{
1527 Cluster: cluster,
1528 }
1529 parent := fmt.Sprintf("projects/%s/locations/%s", projectID, location)
1530 op, err := containerClient.Projects.Locations.Clusters.Create(parent, req).Do()
1531 if err != nil {
1532 return err
1533 }
1534
1535 err = wait.PollImmediate(10*time.Second, 10*time.Minute, func() (done bool, err error) {
1536 name := containerOpFullName(projectID, location, op.Name)
1537 res, err := containerClient.Projects.Locations.Operations.Get(name).Do()
1538 if err != nil {
1539 return false, err
1540 }
1541 if res.Status == "DONE" {
1542 if res.StatusMessage != "" {
1543 return true, fmt.Errorf("cluster creation operation failed: %v", res.StatusMessage)
1544 }
1545 return true, nil
1546 }
1547 log.Info("Waiting for cluster creation operation to finish...")
1548 return false, nil
1549 })
1550 if err != nil {
1551 return err
1552 }
1553
1554 err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (done bool, err error) {
1555 name := clusterFullName(projectID, location, clusterName)
1556 cluster, err := containerClient.Projects.Locations.Clusters.Get(name).Do()
1557 if err != nil {
1558 return false, err
1559 }
1560 if cluster.Status == "RUNNING" {
1561 return true, nil
1562 }
1563 log.Info("Waiting for cluster to be in RUNNING state...", "currentState", cluster.Status)
1564 return false, nil
1565 })
1566 if err != nil {
1567 return err
1568 }
1569 return nil
1570 }
1571
1572 func deleteGKECluster(containerClient *containerBeta.Service, clusterName, projectID, location string) error {
1573 name := clusterFullName(projectID, location, clusterName)
1574 _, err := containerClient.Projects.Locations.Clusters.Delete(name).Do()
1575 if err != nil {
1576 return err
1577 }
1578 return nil
1579 }
1580
1581 func containerOpFullName(projectID, location, opName string) string {
1582 return fmt.Sprintf("projects/%s/locations/%s/operations/%s", projectID, location, opName)
1583 }
1584
1585 func clusterFullName(projectID, location, clusterName string) string {
1586 return fmt.Sprintf("projects/%s/locations/%s/clusters/%s", projectID, location, clusterName)
1587 }
1588
1589 func getKubeconfigForGKECluster(projectID, location, clusterName, outputFilePath string) error {
1590 cmd := exec.Command(
1591 "gcloud", "container", "clusters", "get-credentials", clusterName,
1592 "--zone", location,
1593 "--project", projectID,
1594 )
1595
1596
1597
1598 envVarOverride := "KUBECONFIG=" + outputFilePath
1599 cmd.Env = append(os.Environ(), envVarOverride)
1600 return utils.Execute(cmd)
1601 }
1602
1603 func (k *kubectl) applyStdin(stdin string, args ...string) (string, error) {
1604 return k.exec("apply", stdin, args...)
1605 }
1606
1607 func (k *kubectl) apply(args ...string) (stdout string, err error) {
1608 return k.applyStdin("", args...)
1609 }
1610
1611 func (k *kubectl) delete(args ...string) (stdout string, err error) {
1612 timeout := fmt.Sprintf("%vs", k.deleteTimeout.Seconds())
1613 args = append(args, "--timeout", timeout)
1614 stdout, err = k.exec("delete", "", args...)
1615 if err != nil && strings.Contains(err.Error(), "Error from server (NotFound)") {
1616
1617 return "", nil
1618 }
1619 return stdout, err
1620 }
1621
1622 func (k *kubectl) get(args ...string) (stdout string, err error) {
1623 return k.exec("get", "", args...)
1624 }
1625
1626 func (k *kubectl) exec(command, stdin string, args ...string) (stdout string, err error) {
1627 if k.kubeconfigPath == "" {
1628 return "", fmt.Errorf("attempted to execute a kubectl command without a specified kubeconfig")
1629 }
1630 args = append([]string{command}, args...)
1631 args = append(args, "--kubeconfig", k.kubeconfigPath)
1632 cmd := exec.Command("kubectl", args...)
1633 if stdin != "" {
1634 cmd.Stdin = bytes.NewBufferString(stdin)
1635 }
1636 return utils.ExecuteAndCaptureOutput(cmd)
1637 }
1638
1639 func downloadAndExtractOperatorReleaseTarball(version, outputDir string) error {
1640 tarballGCSPath := fmt.Sprintf("gs://%v/%v/%v", OPERATOR_RELEASE_BUCKET, version, OPERATOR_RELEASE_TARBALL)
1641 return utils.DownloadAndExtractTarballAt(tarballGCSPath, outputDir)
1642 }
1643
1644 func downloadAndExtractKCCReleaseTarball(version, outputDir string) error {
1645 tarballGCSPath := fmt.Sprintf("gs://%v/%v/%v", KCC_RELEASE_BUCKET, version, KCC_RELEASE_TARBALL)
1646 return utils.DownloadAndExtractTarballAt(tarballGCSPath, outputDir)
1647 }
1648
1649 func createTempDir(namePrefix string) (path string, err error) {
1650
1651
1652 return ioutil.TempDir("", namePrefix)
1653 }
1654
1655 func writeToFile(content string, filePath string) error {
1656 fileMode := os.FileMode(0644)
1657 return ioutil.WriteFile(filePath, []byte(content), fileMode)
1658 }
1659
1660 func (c *cluster) createNamespace(namespace string) error {
1661 ns := &v1.Namespace{
1662 ObjectMeta: metav1.ObjectMeta{
1663 Name: namespace,
1664 },
1665 }
1666 _, err := c.clientset.CoreV1().Namespaces().Create(context.Background(), ns, metav1.CreateOptions{})
1667 return err
1668 }
1669
1670 func (c *cluster) deleteNamespace(namespace string) error {
1671 return c.clientset.CoreV1().Namespaces().Delete(context.Background(), namespace, metav1.DeleteOptions{})
1672 }
1673
1674 func (c *cluster) addProjectIDAnnotationToNamespace(namespace, projectID string) error {
1675 getFunc := func() (interface{}, error) {
1676 return c.getNamespace(namespace)
1677 }
1678 res, err := c.retry(getFunc, defaultBackOff)
1679 if err != nil {
1680 return err
1681 }
1682 ns := res.(*v1.Namespace)
1683 annotations := getAnnotationsForNS(ns)
1684 annotations[k8s.ProjectIdAnnotation] = projectID
1685 ns.SetAnnotations(annotations)
1686 updateFunc := func() (interface{}, error) {
1687 ns, err = c.clientset.CoreV1().Namespaces().Update(context.Background(), ns, metav1.UpdateOptions{})
1688 if err != nil {
1689 return nil, fmt.Errorf("error updating namespace '%v': %v", namespace, err)
1690 }
1691 return ns, nil
1692 }
1693 _, err = c.retry(updateFunc, defaultBackOff)
1694 return err
1695 }
1696
1697
1698
1699
1700
1701
1702
1703 func (c *cluster) retry(f func() (interface{}, error), backoff wait.Backoff) (interface{}, error) {
1704 var funcError error
1705 var res interface{}
1706 if err := wait.ExponentialBackoff(backoff, func() (bool, error) {
1707 res, funcError = f()
1708 if funcError != nil {
1709 c.log.Info("Retrying after encountering error", "error", funcError)
1710 return false, nil
1711 }
1712 return true, nil
1713 }); err != nil {
1714 return nil, funcError
1715 }
1716 return res, nil
1717 }
1718
1719 func (c *cluster) getKCCVersion() (string, error) {
1720 ns, err := c.getNamespace(k8s.CNRMSystemNamespace)
1721 if err != nil {
1722 return "", err
1723 }
1724 annotations := getAnnotationsForNS(ns)
1725 version, ok := annotations[k8s.VersionAnnotation]
1726 if !ok {
1727 return "", fmt.Errorf("KCC version annotation ('%v') not found in namespace '%v'", k8s.VersionAnnotation, k8s.CNRMSystemNamespace)
1728 }
1729 return version, nil
1730 }
1731
1732 func (c *cluster) getOperatorVersion() (string, error) {
1733 ns, err := c.getNamespace(k8s.OperatorSystemNamespace)
1734 if err != nil {
1735 return "", err
1736 }
1737 annotations := getAnnotationsForNS(ns)
1738 version, ok := annotations[k8s.OperatorVersionAnnotation]
1739 if !ok {
1740 return "", fmt.Errorf("KCC operator version annotation ('%v') not found in namespace '%v'", k8s.OperatorVersionAnnotation, k8s.OperatorSystemNamespace)
1741 }
1742 return version, nil
1743 }
1744
1745 func (c *cluster) getNamespace(namespace string) (*v1.Namespace, error) {
1746 return c.clientset.CoreV1().Namespaces().Get(context.Background(), namespace, metav1.GetOptions{})
1747 }
1748
1749 func getAnnotationsForNS(ns *v1.Namespace) map[string]string {
1750 annotations := ns.GetAnnotations()
1751 if annotations == nil {
1752 return make(map[string]string)
1753 }
1754 return annotations
1755 }
1756
1757 func newUniqueTestId() string {
1758 return randomid.New().String()
1759 }
1760
1761 func newLogger(name string) (logr.Logger, error) {
1762 zapConfig := zap.NewDevelopmentConfig()
1763 zapConfig.DisableCaller = true
1764 zapLog, err := zapConfig.Build()
1765 if err != nil {
1766 return logr.Discard(), err
1767 }
1768 log := zapr.NewLogger(zapLog)
1769 return log.WithName(name), nil
1770 }
1771
View as plain text