// Copyright 2022 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package testcontroller import ( "context" "fmt" "reflect" "strconv" "strings" "testing" "time" "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/k8s" testgcp "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/gcp" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" ) func DeleteAllEventsForUnstruct(t *testing.T, c client.Client, unstruct *unstructured.Unstructured) { for _, e := range getEventsForObject(t, c, unstruct.GetKind(), unstruct.GetName(), unstruct.GetNamespace()) { if err := c.Delete(context.TODO(), &e); err != nil { t.Fatalf("unable to delete event for %v %v/%v: %v", unstruct.GetKind(), unstruct.GetNamespace(), unstruct.GetName(), err) } } } func AssertEventRecordedForObjectMetaAndKind(t *testing.T, c client.Client, kind string, om *metav1.ObjectMeta, reason string) { assertEventRecorded(t, c, kind, om.Name, om.Namespace, reason) } func AssertEventRecordedforUnstruct(t *testing.T, c client.Client, unstruct *unstructured.Unstructured, reason string) { assertEventRecorded(t, c, unstruct.GetKind(), unstruct.GetName(), unstruct.GetNamespace(), reason) } func AssertEventNotRecordedforUnstruct(t *testing.T, c client.Client, unstruct *unstructured.Unstructured, reason string) { assertEventNotRecorded(t, c, unstruct.GetKind(), unstruct.GetName(), unstruct.GetNamespace(), reason) } func AssertObservedGenerationEquals(t *testing.T, unstruct *unstructured.Unstructured, preReconcileGeneration int64) { observedGeneration, found, err := unstructured.NestedInt64(unstruct.Object, "status", "observedGeneration") if err != nil { t.Errorf("error getting the value for 'status.observedGeneration': %v", err) } if !found { t.Errorf("'status.observedGeneration' is not found") } if observedGeneration != preReconcileGeneration { t.Errorf("observedGeneration %v doesn't match with the pre-reconcile generation %v", observedGeneration, preReconcileGeneration) } } func assertEventRecorded(t *testing.T, c client.Client, kind, name, namespace, reason string) { err := waitUntilEventRecorded(t, c, kind, name, namespace, reason) if err != nil { t.Errorf("event with reason '%v' not recorded for %v %v/%v", reason, kind, namespace, name) } } func assertEventNotRecorded(t *testing.T, c client.Client, kind, name, namespace, reason string) { err := waitUntilEventRecorded(t, c, kind, name, namespace, reason) if err == nil { t.Errorf("expected event with reason '%v' to not be recorded for %v %v/%v, but it was", reason, kind, namespace, name) } else if err != wait.ErrWaitTimeout { t.Errorf("error waiting for event with reason '%v' to be recorded for %v %v/%v: %v", reason, kind, namespace, name, err) } } func waitUntilEventRecorded(t *testing.T, c client.Client, kind, name, namespace, reason string) error { // Event firing is asynchronous, so we need to poll for whether it occurs interval := 10 * time.Second timeout := 1 * time.Minute return wait.PollImmediate(interval, timeout, func() (done bool, err error) { return eventRecorded(t, c, kind, name, namespace, reason), nil }) } func eventRecorded(t *testing.T, c client.Client, kind, name, namespace, reason string) bool { for _, e := range getEventsForObject(t, c, kind, name, namespace) { if e.Reason == reason { return true } } return false } func getEventsForObject(t *testing.T, c client.Client, kind, name, namespace string) []v1.Event { listOptions := client.ListOptions{ Namespace: namespace, } events := make([]v1.Event, 0) for ok := true; ok; ok = listOptions.Continue != "" { var eventList v1.EventList if err := c.List(context.TODO(), &eventList, &listOptions); err != nil { t.Fatalf("error listing events for %v %v/%v: %v", kind, namespace, name, err) } for _, e := range eventList.Items { obj := &e.InvolvedObject if (obj.Kind == kind) && (obj.Namespace == namespace) && (obj.Name == name) { events = append(events, e) } } listOptions.Continue = eventList.Continue } return events } func WaitForUnstructDeleteToFinish(t *testing.T, kubeClient client.Client, origUnstruct *unstructured.Unstructured) { unstruct := origUnstruct.DeepCopy() err := wait.PollImmediate(1*time.Second, 30*time.Second, func() (done bool, err error) { err = kubeClient.Get(context.TODO(), k8s.GetNamespacedName(unstruct), unstruct) if err == nil { return false, nil } if errors.IsNotFound(err) { return true, nil } return true, err }) if err != nil { t.Fatalf("error waiting for %v %v/%v to be deleted: %v", unstruct.GetKind(), unstruct.GetNamespace(), unstruct.GetName(), err) } } // ReplaceTestVars replaces all occurrences of placeholder strings e.g. ${uniqueId} in a given byte slice. func ReplaceTestVars(t *testing.T, b []byte, uniqueId string, project testgcp.GCPProject) []byte { s := string(b) s = strings.Replace(s, "${uniqueId}", uniqueId, -1) s = strings.Replace(s, "${projectId}", project.ProjectID, -1) if strings.Contains(s, "${projectNumber}") { projectNumber := strconv.FormatInt(project.ProjectNumber, 10) s = strings.Replace(s, "${projectNumber}", projectNumber, -1) } // Handle placeholder strings for folder id and org id specially because they are pure numbers while yaml marshalling expects strings. s = strings.Replace(s, fmt.Sprintf("folders/${%s}", testgcp.TestFolderId), fmt.Sprintf("folders/%s", testgcp.GetFolderID(t)), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.TestFolderId), fmt.Sprintf("\"%s\"", testgcp.GetFolderID(t)), -1) s = strings.Replace(s, fmt.Sprintf("folders/${%s}", testgcp.TestFolder2Id), fmt.Sprintf("folders/%s", testgcp.GetFolder2ID(t)), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.TestFolder2Id), fmt.Sprintf("\"%s\"", testgcp.GetFolder2ID(t)), -1) s = strings.Replace(s, fmt.Sprintf("organizations/${%s}", testgcp.TestOrgId), fmt.Sprintf("organizations/%s", testgcp.GetOrgID(t)), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.TestOrgId), fmt.Sprintf("\"%s\"", testgcp.GetOrgID(t)), -1) s = strings.Replace(s, fmt.Sprintf("projects/${%s}", testgcp.TestDependentOrgProjectId), fmt.Sprintf("projects/%s", testgcp.GetDependentOrgProjectID(t)), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.TestDependentOrgProjectId), fmt.Sprintf("\"%s\"", testgcp.GetDependentOrgProjectID(t)), -1) s = strings.Replace(s, fmt.Sprintf("projects/${%s}", testgcp.TestDependentFolderProjectId), fmt.Sprintf("projects/%s", testgcp.GetDependentFolderProjectID(t)), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.TestDependentFolderProjectId), fmt.Sprintf("\"%s\"", testgcp.GetDependentFolderProjectID(t)), -1) s = strings.Replace(s, fmt.Sprintf("projects/${%s}", testgcp.TestDependentNoNetworkProjectId), fmt.Sprintf("projects/%s", testgcp.GetDependentNoNetworkProjectID(t)), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.TestDependentNoNetworkProjectId), fmt.Sprintf("\"%s\"", testgcp.GetDependentNoNetworkProjectID(t)), -1) s = strings.Replace(s, fmt.Sprintf("organizations/${%s}", testgcp.IAMIntegrationTestsOrganizationId), fmt.Sprintf("organizations/%s", testgcp.GetIAMIntegrationTestsOrganizationId(t)), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.IAMIntegrationTestsOrganizationId), fmt.Sprintf("\"%s\"", testgcp.GetIAMIntegrationTestsOrganizationId(t)), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.IsolatedTestOrgName), testgcp.GetIsolatedTestOrgName(t), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.TestBillingAccountId), testgcp.GetBillingAccountID(t), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.TestBillingAccountIDForBillingResources), testgcp.GetTestBillingAccountIDForBillingResources(t), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.IAMIntegrationTestsBillingAccountId), testgcp.GetIAMIntegrationTestsBillingAccountId(t), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.FirestoreTestProject), testgcp.GetFirestoreTestProject(t), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.CloudFunctionsTestProject), testgcp.GetCloudFunctionsTestProject(t), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.IdentityPlatformTestProject), testgcp.GetIdentityPlatformTestProject(t), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.InterconnectTestProject), testgcp.GetInterconnectTestProject(t), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.HighCPUQuotaTestProject), testgcp.GetHighCpuQuotaTestProject(t), -1) s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.RecaptchaEnterpriseTestProject), testgcp.GetRecaptchaEnterpriseTestProject(t), -1) return []byte(s) } // Collects an expected number of events from the API server. The timeout is applied on a per-event basis, so it is possible this function // takes upwards of expectedCount * timeoutSeconds duration to 'timeout'. When a timeout does occur, t.Fatal(...) is invoked. func CollectEvents(t *testing.T, config *rest.Config, namespace string, expectedCount int, timeout time.Duration) []v1.Event { t.Helper() clientSet, err := kubernetes.NewForConfig(config) if err != nil { t.Fatalf("error creating k8s client: %v", err) } listOptions := metav1.ListOptions{} watcher, err := clientSet.CoreV1().Events(namespace).Watch(context.Background(), listOptions) if err != nil { t.Fatalf("errror creating event watch: %v", err) } defer watcher.Stop() results := make([]v1.Event, 0) ch := watcher.ResultChan() for i := 0; i < expectedCount; i++ { select { case res := <-ch: event, ok := res.Object.(*v1.Event) if !ok { t.Fatalf("unexpected type returned in channel: got '%v', watch '%v'", reflect.TypeOf(res), reflect.TypeOf(v1.Event{})) } results = append(results, *event) case <-time.After(timeout): t.Fatalf("expected '%v' event(s), collected '%v' event(s), timed out waiting for the last '%v' event(s)t'", expectedCount, len(results), expectedCount-len(results)) } } return results }