kccmanager_test.go

Documentation: github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/kccmanager

     1  // Copyright 2022 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  //go:build integration
    16  // +build integration
    17  
    18  package kccmanager_test
    19  
    20  import (
    21  	"context"
    22  	"fmt"
    23  	"testing"
    24  	"time"
    25  
    26  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/dynamic"
    27  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/kccmanager"
    28  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test"
    29  	testcontroller "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/controller"
    30  	testgcp "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/gcp"
    31  	testmain "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/main"
    32  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/resourcefixture"
    33  	testrunner "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/runner"
    34  
    35  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    36  	"k8s.io/apimachinery/pkg/runtime"
    37  	"k8s.io/apimachinery/pkg/types"
    38  	"k8s.io/apimachinery/pkg/util/wait"
    39  	"k8s.io/klog/v2"
    40  	"sigs.k8s.io/controller-runtime/pkg/client"
    41  	"sigs.k8s.io/controller-runtime/pkg/manager"
    42  )
    43  
    44  var (
    45  	// These managers are used to just get the rest.Config since our testmain package's methods are not easily changed
    46  	// to return a rest.Config
    47  	clusterModeManager    manager.Manager
    48  	namespacedModeManager manager.Manager
    49  )
    50  
    51  // The scheme is not thread-safe due to its use and modification of its internal maps. Different managers should not
    52  // share a scheme.
    53  func TestSchemeIsUniqueAcrossManagers(t *testing.T) {
    54  	ctx := context.TODO()
    55  
    56  	controllersCfg := kccmanager.Config{
    57  		ManagerOptions: manager.Options{
    58  			// disable prometheus metrics as by default, the metrics server binds to the same port in all instances
    59  			MetricsBindAddress: "0",
    60  		},
    61  	}
    62  	schemePtrMap := make(map[*runtime.Scheme]string)
    63  	schemePtrMap[clusterModeManager.GetScheme()] = "clusterModeMgr"
    64  	for i := 0; i < 5; i++ {
    65  		mgr, err := kccmanager.New(ctx, clusterModeManager.GetConfig(), controllersCfg)
    66  		if err != nil {
    67  			t.Fatalf("error creating manager: %v", err)
    68  		}
    69  		mgrName := fmt.Sprintf("mgr-%v", i)
    70  		if val, ok := schemePtrMap[mgr.GetScheme()]; ok {
    71  			t.Fatalf("expected new manager '%v' to have a new, unique scheme, instead it is sharing the scheme with '%v'", mgrName, val)
    72  		}
    73  		schemePtrMap[mgr.GetScheme()] = mgrName
    74  	}
    75  }
    76  
    77  func TestClusterModeManager(t *testing.T) {
    78  	ctx := context.TODO()
    79  	mgr, err := kccmanager.New(ctx, clusterModeManager.GetConfig(), kccmanager.Config{})
    80  	if err != nil {
    81  		t.Fatalf("error creating manager: %v", err)
    82  	}
    83  	stop := testcontroller.StartMgr(t, mgr)
    84  	defer stop()
    85  	basicPubSubFixture := getBasicPubSubSchemaFixture(t)
    86  	project := testgcp.GetDefaultProject(t)
    87  	for i := 0; i < 2; i++ {
    88  		tstContext := testrunner.NewTestContext(t, basicPubSubFixture, project)
    89  		testcontroller.EnsureNamespaceExistsT(t, mgr.GetClient(), tstContext.CreateUnstruct.GetNamespace())
    90  		if err := mgr.GetClient().Create(context.TODO(), tstContext.CreateUnstruct); err != nil {
    91  			t.Fatalf("error creating '%v': %v", tstContext.CreateUnstruct.GetKind(), err)
    92  		}
    93  		waitForReconcile(t, mgr.GetClient(), tstContext.CreateUnstruct)
    94  	}
    95  }
    96  
    97  // Create two resources, one in a managed namespace for which we have started controllers, and another for which we have
    98  // not started controllers. Verify that only the first is reconciled, then start a second set of controllers and verify
    99  // the second is reconciled.
   100  func TestNamespacedModeManager(t *testing.T) {
   101  	ctx := context.TODO()
   102  	basicPubSubFixture := getBasicPubSubSchemaFixture(t)
   103  	project := testgcp.GetDefaultProject(t)
   104  	tstContext1 := testrunner.NewTestContext(t, basicPubSubFixture, project)
   105  	tstContext2 := testrunner.NewTestContext(t, basicPubSubFixture, project)
   106  	controllersCfg1 := kccmanager.Config{
   107  		ManagerOptions: manager.Options{
   108  			// disable prometheus metrics as by default, the metrics server binds to the same port in all instances
   109  			MetricsBindAddress: "0",
   110  			Namespace:          tstContext1.CreateUnstruct.GetNamespace(),
   111  		},
   112  	}
   113  	mgr1, err := kccmanager.New(ctx, namespacedModeManager.GetConfig(), controllersCfg1)
   114  	if err != nil {
   115  		t.Fatalf("error creating manager: %v", err)
   116  	}
   117  	testcontroller.StartMgr(t, mgr1)
   118  	// TODO: delete the line above and uncomment the two lines below once we have a fix for the race condition out of
   119  	// client-go, sollyross@ is working on it: https://github.com/kubernetes/kubernetes/pull/95664/files
   120  	//stop1 := testcontroller.StartMgr(t, mgr1)
   121  	//defer stop1()
   122  	kubeClient := namespacedModeManager.GetClient()
   123  	testcontroller.EnsureNamespaceExistsT(t, kubeClient, tstContext1.CreateUnstruct.GetNamespace())
   124  	if err := kubeClient.Create(context.TODO(), tstContext1.CreateUnstruct); err != nil {
   125  		t.Fatalf("error creating '%v': %v", tstContext1.CreateUnstruct.GetKind(), err)
   126  	}
   127  	testcontroller.EnsureNamespaceExistsT(t, kubeClient, tstContext2.CreateUnstruct.GetNamespace())
   128  	if err := kubeClient.Create(context.TODO(), tstContext2.CreateUnstruct); err != nil {
   129  		t.Fatalf("error creating '%v': %v", tstContext2.CreateUnstruct.GetKind(), err)
   130  	}
   131  	waitForReconcile(t, kubeClient, tstContext1.CreateUnstruct)
   132  	// sleep 10 seconds to give this resource 'time to reconcile' even though we expect it to NOT reconcile at all as
   133  	// no controllers are running
   134  	time.Sleep(10 * time.Second)
   135  	if err := kubeClient.Get(context.TODO(), tstContext2.NamespacedName, tstContext2.CreateUnstruct); err != nil {
   136  		t.Fatalf("error getting resource: %v", err)
   137  	}
   138  	var expectedValue interface{} = nil
   139  	actualValue := tstContext2.CreateUnstruct.Object["status"]
   140  	if actualValue != expectedValue {
   141  		t.Fatalf("unexpected value for status: got '%v', want '%v'", actualValue, expectedValue)
   142  	}
   143  	controllersCfg2 := kccmanager.Config{
   144  		ManagerOptions: manager.Options{
   145  			// disable prometheus metrics as by default, the metrics server binds to the same port in all instances
   146  			MetricsBindAddress: "0",
   147  			Namespace:          tstContext2.CreateUnstruct.GetNamespace(),
   148  		},
   149  	}
   150  	// start controllers for the second namespace and verify that the second resource does reconcile
   151  	mgr2, err := kccmanager.New(ctx, namespacedModeManager.GetConfig(), controllersCfg2)
   152  	if err != nil {
   153  		t.Fatalf("error creating manager: %v", err)
   154  	}
   155  	testcontroller.StartMgr(t, mgr2)
   156  	// TODO: delete the line above and uncomment the two lines below once we have a fix for the race condition out of
   157  	// client-go, sollyross@ is working on it: https://github.com/kubernetes/kubernetes/pull/95664/files
   158  	//stop2 := testcontroller.StartMgr(t, mgr2)
   159  	//defer stop2()
   160  	waitForReconcile(t, kubeClient, tstContext2.CreateUnstruct)
   161  }
   162  
   163  // getBasicPubSubSchemaFixture returns the basic/pubsubschema fixture.
   164  // This is a relatively quick resource to create, that does not have any dependencies that must be created.
   165  func getBasicPubSubSchemaFixture(t *testing.T) resourcefixture.ResourceFixture {
   166  	lightFilter := func(name string, testType resourcefixture.TestType) bool {
   167  		return name == "pubsubschema" && testType == resourcefixture.Basic
   168  	}
   169  	fixtures := resourcefixture.LoadWithFilter(t, lightFilter, nil)
   170  	if len(fixtures) != 1 {
   171  		t.Fatalf("unexpected number of fixtures: got '%v', want '%v'", len(fixtures), 1)
   172  	}
   173  	return fixtures[0]
   174  }
   175  
   176  func waitForReconcile(t *testing.T, kubeClient client.Client, resource *unstructured.Unstructured) {
   177  	// return value of true means 'done'
   178  	condFunc := func() (bool, error) {
   179  		nn := types.NamespacedName{
   180  			Namespace: resource.GetNamespace(),
   181  			Name:      resource.GetName(),
   182  		}
   183  		u := unstructured.Unstructured{}
   184  		u.SetGroupVersionKind(resource.GroupVersionKind())
   185  		if err := kubeClient.Get(context.TODO(), nn, &u); err != nil {
   186  			return false, fmt.Errorf("error getting '%v': %v", nn, err)
   187  		}
   188  		if u.Object["status"] == nil {
   189  			klog.Infof("Waiting for 'status' on %v '%v'", u.GetKind(), u.GetName())
   190  			return false, nil
   191  		}
   192  		conditions := dynamic.GetConditions(t, &u)
   193  		if len(conditions) == 0 {
   194  			return false, nil
   195  		}
   196  		return true, nil
   197  	}
   198  	if err := wait.PollImmediate(10*time.Second, 5*time.Minute, condFunc); err != nil {
   199  		t.Fatalf("error waiting for reconcile of '%v' to complete: %v'", resource.GetKind(), err)
   200  	}
   201  }
   202  
   203  func TestMain(m *testing.M) {
   204  	managers := []*manager.Manager{
   205  		&clusterModeManager,
   206  		&namespacedModeManager,
   207  	}
   208  	testmain.TestMainSetupMultipleEnvironments(m, test.IntegrationTestType, nil, managers)
   209  }
   210
View as plain text