resourcecontention_integration_test.go

Documentation: github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourcecontention

     1  // Copyright 2022 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  //go:build integration
    16  // +build integration
    17  
    18  package resourcecontention_test
    19  
    20  import (
    21  	"context"
    22  	"regexp"
    23  	"testing"
    24  	"time"
    25  
    26  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/cluster"
    27  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/k8s"
    28  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/lease/leaser"
    29  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test"
    30  	testcontroller "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/controller"
    31  	testreconciler "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/controller/reconciler"
    32  	testgcp "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/gcp"
    33  	testmain "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/main"
    34  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/resourcefixture"
    35  	testrunner "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/runner"
    36  
    37  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    38  	"k8s.io/apimachinery/pkg/types"
    39  	"sigs.k8s.io/controller-runtime/pkg/manager"
    40  )
    41  
    42  var (
    43  	mgr1 manager.Manager
    44  	mgr2 manager.Manager
    45  )
    46  
    47  // this test creates a second 'manager', i.e. KCC installation, and then ensures that the same project
    48  // that is being managed by the default manager in systemContext, cannot be managed by the second manager.
    49  func TestResourceContentionIsPreventedForTwoNamespacesMappingToSameProjectInDifferentClusters(t *testing.T) {
    50  	shouldRun := func(fixture resourcefixture.ResourceFixture, mgr manager.Manager) bool {
    51  		// only need to test contention for a single resource since the logic will apply to all resources
    52  		return fixture.GVK.Kind == "PubSubTopic"
    53  	}
    54  	testFunc := func(t *testing.T, testContext testrunner.TestContext, systemContext testrunner.SystemContext) {
    55  		if err := systemContext.Manager.GetClient().Create(context.TODO(), testContext.CreateUnstruct); err != nil {
    56  			t.Fatalf("error creating resource: %v", err)
    57  		}
    58  		systemContext.Reconciler.Reconcile(testContext.UpdateUnstruct, testreconciler.ExpectedSuccessfulReconcileResultFor(systemContext.Reconciler, testContext.UpdateUnstruct), nil)
    59  		assertLeaseLabelsAreNotPresent(t, systemContext.Manager, testContext.CreateUnstruct)
    60  		projectId := testgcp.GetDefaultProjectID(t)
    61  		testcontroller.EnsureNamespaceExistsT(t, mgr2.GetClient(), testContext.UniqueId)
    62  		testcontroller.EnsureNamespaceHasProjectIDAnnotation(t, mgr2.GetClient(), testContext.UniqueId, projectId)
    63  		assertNamespaceIdsAreNotEqual(t, systemContext.Manager, mgr2, testContext.UniqueId, testContext.UniqueId)
    64  		reconciler2 := testreconciler.New(t, mgr2, systemContext.TFProvider)
    65  		if err := mgr2.GetClient().Create(context.TODO(), testContext.UpdateUnstruct); err != nil {
    66  			t.Fatalf("error creating resource: %v", err)
    67  		}
    68  		reconciler2.Reconcile(testContext.UpdateUnstruct, testreconciler.ExpectedUnsuccessfulReconcileResult, regexp.MustCompile("error obtaining lease"))
    69  		events := testcontroller.CollectEvents(t, mgr2.GetConfig(), testContext.UpdateUnstruct.GetNamespace(), 1, 10*time.Second)
    70  		event := events[0]
    71  		expectedReason := k8s.ManagementConflict
    72  		if event.Reason != expectedReason {
    73  			t.Fatalf("event mismatch: got '%v', want '%v'", event.Reason, expectedReason)
    74  		}
    75  		// Since the controller was unable to obtain the lease it does not write the default finalizer onto the object.
    76  		// Add the finalizer manually so that we can test the deletion resource contention flow.
    77  		ensureFinalizer(t, mgr2, testContext.NamespacedName, testContext.CreateUnstruct)
    78  		if err := mgr2.GetClient().Delete(context.TODO(), testContext.CreateUnstruct); err != nil {
    79  			t.Fatalf("error deleting resource: %v", err)
    80  		}
    81  		reconciler2.Reconcile(testContext.CreateUnstruct, testreconciler.ExpectedUnsuccessfulReconcileResult, regexp.MustCompile("error obtaining lease"))
    82  		events = testcontroller.CollectEvents(t, mgr2.GetConfig(), testContext.CreateUnstruct.GetNamespace(), 3, 10*time.Second)
    83  		nextEvent := events[2]
    84  		if nextEvent.Reason != expectedReason {
    85  			t.Fatalf("event mismatch: got '%v', want '%v'", nextEvent.Reason, expectedReason)
    86  		}
    87  		if !(event.LastTimestamp == nextEvent.LastTimestamp || event.LastTimestamp.Before(&nextEvent.LastTimestamp)) {
    88  			t.Fatalf("expected the previous event's last timestamp to be before or equal to the next event's last timestamp")
    89  		}
    90  	}
    91  	testrunner.RunAllWithDependenciesCreatedButNotObject(t, mgr1, shouldRun, testFunc)
    92  }
    93  
    94  func ensureFinalizer(t *testing.T, mgr manager.Manager, namespacedName types.NamespacedName, u *unstructured.Unstructured) {
    95  	if err := mgr.GetClient().Get(context.TODO(), namespacedName, u); err != nil {
    96  		t.Fatalf("error getting '%v': %v", namespacedName, err)
    97  	}
    98  	if k8s.EnsureFinalizer(u, k8s.ControllerFinalizerName) {
    99  		t.Fatalf("found a finalizer when none was expected")
   100  	}
   101  	if err := mgr.GetClient().Update(context.TODO(), u); err != nil {
   102  		t.Fatalf("error updating '%v': %v", namespacedName, err)
   103  	}
   104  }
   105  
   106  func assertLeaseLabelsAreNotPresent(t *testing.T, mgr manager.Manager, u *unstructured.Unstructured) {
   107  	name := k8s.GetNamespacedName(u)
   108  	if err := mgr.GetClient().Get(context.TODO(), name, u); err != nil {
   109  		t.Fatalf("error getting resource '%v': %v", name, err)
   110  	}
   111  	if u.GetLabels() == nil {
   112  		return
   113  	}
   114  	// the keys are hard-coded here to make sure the test doesn't have any knowledge of the implementation
   115  	leaseKeys := []string{"cnrm-lease-expiration", "cnrm-lease-holder-id"}
   116  	// add a check to make sure the implementation has not changed without the test being updated
   117  	assertLeaseKeysMatch(t, leaseKeys)
   118  	for _, k := range leaseKeys {
   119  		if val, ok := u.GetLabels()[k]; ok {
   120  			t.Fatalf("unexpected value of '%v' for label key '%v': the key should not be present", k, val)
   121  		}
   122  	}
   123  }
   124  
   125  func assertLeaseKeysMatch(t *testing.T, expectedKeys []string) {
   126  	keys := leaser.GetLabelKeys()
   127  	keysMap := make(map[string]bool)
   128  	for _, k := range keys {
   129  		keysMap[k] = true
   130  	}
   131  	for _, k := range expectedKeys {
   132  		if _, ok := keysMap[k]; !ok {
   133  			t.Fatalf("missing expected key '%v'", k)
   134  		}
   135  	}
   136  }
   137  
   138  func assertNamespaceIdsAreNotEqual(t *testing.T, mgr1, mgr2 manager.Manager, namespace1, namespace2 string) {
   139  	id1 := getNamespaceID(t, mgr1, namespace1)
   140  	id2 := getNamespaceID(t, mgr2, namespace2)
   141  	if id1 == id2 {
   142  		t.Fatalf("expected the ids of both managers to not match, instead, both have an id of '%v'", id1)
   143  	}
   144  }
   145  
   146  func getNamespaceID(t *testing.T, mgr manager.Manager, namespace string) string {
   147  	t.Helper()
   148  	id, err := cluster.GetNamespaceID(k8s.NamespaceIDConfigMapNN, mgr.GetClient(), context.TODO(), namespace)
   149  	if err != nil {
   150  		t.Fatal(err)
   151  	}
   152  	return id
   153  }
   154  
   155  func TestMain(m *testing.M) {
   156  	mgrs := []*manager.Manager{&mgr1, &mgr2}
   157  	testmain.TestMainSetupMultipleEnvironments(m, test.IntegrationTestType, nil, mgrs)
   158  }
   159
View as plain text