k8s.go

Documentation: github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/controller

     1  // Copyright 2022 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package testcontroller
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"reflect"
    21  	"strconv"
    22  	"strings"
    23  	"testing"
    24  	"time"
    25  
    26  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/k8s"
    27  	testgcp "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/gcp"
    28  
    29  	v1 "k8s.io/api/core/v1"
    30  	"k8s.io/apimachinery/pkg/api/errors"
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    33  	"k8s.io/apimachinery/pkg/util/wait"
    34  	"k8s.io/client-go/kubernetes"
    35  	"k8s.io/client-go/rest"
    36  	"sigs.k8s.io/controller-runtime/pkg/client"
    37  )
    38  
    39  func DeleteAllEventsForUnstruct(t *testing.T, c client.Client, unstruct *unstructured.Unstructured) {
    40  	for _, e := range getEventsForObject(t, c, unstruct.GetKind(), unstruct.GetName(), unstruct.GetNamespace()) {
    41  		if err := c.Delete(context.TODO(), &e); err != nil {
    42  			t.Fatalf("unable to delete event for %v %v/%v: %v", unstruct.GetKind(), unstruct.GetNamespace(), unstruct.GetName(), err)
    43  		}
    44  	}
    45  }
    46  
    47  func AssertEventRecordedForObjectMetaAndKind(t *testing.T, c client.Client, kind string, om *metav1.ObjectMeta, reason string) {
    48  	assertEventRecorded(t, c, kind, om.Name, om.Namespace, reason)
    49  }
    50  
    51  func AssertEventRecordedforUnstruct(t *testing.T, c client.Client, unstruct *unstructured.Unstructured, reason string) {
    52  	assertEventRecorded(t, c, unstruct.GetKind(), unstruct.GetName(), unstruct.GetNamespace(), reason)
    53  }
    54  
    55  func AssertEventNotRecordedforUnstruct(t *testing.T, c client.Client, unstruct *unstructured.Unstructured, reason string) {
    56  	assertEventNotRecorded(t, c, unstruct.GetKind(), unstruct.GetName(), unstruct.GetNamespace(), reason)
    57  }
    58  
    59  func AssertObservedGenerationEquals(t *testing.T, unstruct *unstructured.Unstructured, preReconcileGeneration int64) {
    60  	observedGeneration, found, err := unstructured.NestedInt64(unstruct.Object, "status", "observedGeneration")
    61  	if err != nil {
    62  		t.Errorf("error getting the value for 'status.observedGeneration': %v", err)
    63  	}
    64  	if !found {
    65  		t.Errorf("'status.observedGeneration' is not found")
    66  	}
    67  	if observedGeneration != preReconcileGeneration {
    68  		t.Errorf("observedGeneration %v doesn't match with the pre-reconcile generation %v", observedGeneration, preReconcileGeneration)
    69  	}
    70  }
    71  
    72  func assertEventRecorded(t *testing.T, c client.Client, kind, name, namespace, reason string) {
    73  	err := waitUntilEventRecorded(t, c, kind, name, namespace, reason)
    74  	if err != nil {
    75  		t.Errorf("event with reason '%v' not recorded for %v %v/%v", reason, kind, namespace, name)
    76  	}
    77  }
    78  
    79  func assertEventNotRecorded(t *testing.T, c client.Client, kind, name, namespace, reason string) {
    80  	err := waitUntilEventRecorded(t, c, kind, name, namespace, reason)
    81  	if err == nil {
    82  		t.Errorf("expected event with reason '%v' to not be recorded for %v %v/%v, but it was", reason, kind, namespace, name)
    83  	} else if err != wait.ErrWaitTimeout {
    84  		t.Errorf("error waiting for event with reason '%v' to be recorded for %v %v/%v: %v", reason, kind, namespace, name, err)
    85  	}
    86  }
    87  
    88  func waitUntilEventRecorded(t *testing.T, c client.Client, kind, name, namespace, reason string) error {
    89  	// Event firing is asynchronous, so we need to poll for whether it occurs
    90  	interval := 10 * time.Second
    91  	timeout := 1 * time.Minute
    92  	return wait.PollImmediate(interval, timeout, func() (done bool, err error) {
    93  		return eventRecorded(t, c, kind, name, namespace, reason), nil
    94  	})
    95  }
    96  
    97  func eventRecorded(t *testing.T, c client.Client, kind, name, namespace, reason string) bool {
    98  	for _, e := range getEventsForObject(t, c, kind, name, namespace) {
    99  		if e.Reason == reason {
   100  			return true
   101  		}
   102  	}
   103  	return false
   104  }
   105  
   106  func getEventsForObject(t *testing.T, c client.Client, kind, name, namespace string) []v1.Event {
   107  	listOptions := client.ListOptions{
   108  		Namespace: namespace,
   109  	}
   110  	events := make([]v1.Event, 0)
   111  	for ok := true; ok; ok = listOptions.Continue != "" {
   112  		var eventList v1.EventList
   113  		if err := c.List(context.TODO(), &eventList, &listOptions); err != nil {
   114  			t.Fatalf("error listing events for %v %v/%v: %v", kind, namespace, name, err)
   115  		}
   116  		for _, e := range eventList.Items {
   117  			obj := &e.InvolvedObject
   118  			if (obj.Kind == kind) && (obj.Namespace == namespace) && (obj.Name == name) {
   119  				events = append(events, e)
   120  			}
   121  		}
   122  		listOptions.Continue = eventList.Continue
   123  	}
   124  	return events
   125  }
   126  
   127  func WaitForUnstructDeleteToFinish(t *testing.T, kubeClient client.Client, origUnstruct *unstructured.Unstructured) {
   128  	unstruct := origUnstruct.DeepCopy()
   129  	err := wait.PollImmediate(1*time.Second, 30*time.Second, func() (done bool, err error) {
   130  		err = kubeClient.Get(context.TODO(), k8s.GetNamespacedName(unstruct), unstruct)
   131  		if err == nil {
   132  			return false, nil
   133  		}
   134  		if errors.IsNotFound(err) {
   135  			return true, nil
   136  		}
   137  		return true, err
   138  	})
   139  	if err != nil {
   140  		t.Fatalf("error waiting for %v %v/%v to be deleted: %v", unstruct.GetKind(), unstruct.GetNamespace(), unstruct.GetName(), err)
   141  	}
   142  }
   143  
   144  // ReplaceTestVars replaces all occurrences of placeholder strings e.g. ${uniqueId} in a given byte slice.
   145  func ReplaceTestVars(t *testing.T, b []byte, uniqueId string, project testgcp.GCPProject) []byte {
   146  	s := string(b)
   147  	s = strings.Replace(s, "${uniqueId}", uniqueId, -1)
   148  	s = strings.Replace(s, "${projectId}", project.ProjectID, -1)
   149  	if strings.Contains(s, "${projectNumber}") {
   150  		projectNumber := strconv.FormatInt(project.ProjectNumber, 10)
   151  		s = strings.Replace(s, "${projectNumber}", projectNumber, -1)
   152  	}
   153  	// Handle placeholder strings for folder id and org id specially because they are pure numbers while yaml marshalling expects strings.
   154  	s = strings.Replace(s, fmt.Sprintf("folders/${%s}", testgcp.TestFolderId), fmt.Sprintf("folders/%s", testgcp.GetFolderID(t)), -1)
   155  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.TestFolderId), fmt.Sprintf("\"%s\"", testgcp.GetFolderID(t)), -1)
   156  	s = strings.Replace(s, fmt.Sprintf("folders/${%s}", testgcp.TestFolder2Id), fmt.Sprintf("folders/%s", testgcp.GetFolder2ID(t)), -1)
   157  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.TestFolder2Id), fmt.Sprintf("\"%s\"", testgcp.GetFolder2ID(t)), -1)
   158  	s = strings.Replace(s, fmt.Sprintf("organizations/${%s}", testgcp.TestOrgId), fmt.Sprintf("organizations/%s", testgcp.GetOrgID(t)), -1)
   159  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.TestOrgId), fmt.Sprintf("\"%s\"", testgcp.GetOrgID(t)), -1)
   160  	s = strings.Replace(s, fmt.Sprintf("projects/${%s}", testgcp.TestDependentOrgProjectId), fmt.Sprintf("projects/%s", testgcp.GetDependentOrgProjectID(t)), -1)
   161  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.TestDependentOrgProjectId), fmt.Sprintf("\"%s\"", testgcp.GetDependentOrgProjectID(t)), -1)
   162  	s = strings.Replace(s, fmt.Sprintf("projects/${%s}", testgcp.TestDependentFolderProjectId), fmt.Sprintf("projects/%s", testgcp.GetDependentFolderProjectID(t)), -1)
   163  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.TestDependentFolderProjectId), fmt.Sprintf("\"%s\"", testgcp.GetDependentFolderProjectID(t)), -1)
   164  	s = strings.Replace(s, fmt.Sprintf("projects/${%s}", testgcp.TestDependentNoNetworkProjectId), fmt.Sprintf("projects/%s", testgcp.GetDependentNoNetworkProjectID(t)), -1)
   165  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.TestDependentNoNetworkProjectId), fmt.Sprintf("\"%s\"", testgcp.GetDependentNoNetworkProjectID(t)), -1)
   166  	s = strings.Replace(s, fmt.Sprintf("organizations/${%s}", testgcp.IAMIntegrationTestsOrganizationId), fmt.Sprintf("organizations/%s", testgcp.GetIAMIntegrationTestsOrganizationId(t)), -1)
   167  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.IAMIntegrationTestsOrganizationId), fmt.Sprintf("\"%s\"", testgcp.GetIAMIntegrationTestsOrganizationId(t)), -1)
   168  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.IsolatedTestOrgName), testgcp.GetIsolatedTestOrgName(t), -1)
   169  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.TestBillingAccountId), testgcp.GetBillingAccountID(t), -1)
   170  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.TestBillingAccountIDForBillingResources), testgcp.GetTestBillingAccountIDForBillingResources(t), -1)
   171  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.IAMIntegrationTestsBillingAccountId), testgcp.GetIAMIntegrationTestsBillingAccountId(t), -1)
   172  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.FirestoreTestProject), testgcp.GetFirestoreTestProject(t), -1)
   173  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.CloudFunctionsTestProject), testgcp.GetCloudFunctionsTestProject(t), -1)
   174  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.IdentityPlatformTestProject), testgcp.GetIdentityPlatformTestProject(t), -1)
   175  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.InterconnectTestProject), testgcp.GetInterconnectTestProject(t), -1)
   176  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.HighCPUQuotaTestProject), testgcp.GetHighCpuQuotaTestProject(t), -1)
   177  	s = strings.Replace(s, fmt.Sprintf("${%s}", testgcp.RecaptchaEnterpriseTestProject), testgcp.GetRecaptchaEnterpriseTestProject(t), -1)
   178  	return []byte(s)
   179  }
   180  
   181  // Collects an expected number of events from the API server. The timeout is applied on a per-event basis, so it is possible this function
   182  // takes upwards of expectedCount * timeoutSeconds duration to 'timeout'. When a timeout does occur, t.Fatal(...) is invoked.
   183  func CollectEvents(t *testing.T, config *rest.Config, namespace string, expectedCount int, timeout time.Duration) []v1.Event {
   184  	t.Helper()
   185  	clientSet, err := kubernetes.NewForConfig(config)
   186  	if err != nil {
   187  		t.Fatalf("error creating k8s client: %v", err)
   188  	}
   189  	listOptions := metav1.ListOptions{}
   190  	watcher, err := clientSet.CoreV1().Events(namespace).Watch(context.Background(), listOptions)
   191  	if err != nil {
   192  		t.Fatalf("errror creating event watch: %v", err)
   193  	}
   194  	defer watcher.Stop()
   195  	results := make([]v1.Event, 0)
   196  	ch := watcher.ResultChan()
   197  	for i := 0; i < expectedCount; i++ {
   198  		select {
   199  		case res := <-ch:
   200  			event, ok := res.Object.(*v1.Event)
   201  			if !ok {
   202  				t.Fatalf("unexpected type returned in channel: got '%v', watch '%v'", reflect.TypeOf(res), reflect.TypeOf(v1.Event{}))
   203  			}
   204  			results = append(results, *event)
   205  		case <-time.After(timeout):
   206  			t.Fatalf("expected '%v' event(s), collected '%v' event(s), timed out waiting for the last '%v' event(s)t'",
   207  				expectedCount, len(results), expectedCount-len(results))
   208  		}
   209  	}
   210  	return results
   211  }
   212
View as plain text