testreconciler.go

Documentation: github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/controller/reconciler

     1  // Copyright 2022 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package testreconciler
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"log"
    21  	"regexp"
    22  	"testing"
    23  	"time"
    24  
    25  	dclcontroller "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/dcl"
    26  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/iam/auditconfig"
    27  	partialpolicy "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/iam/partialpolicy"
    28  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/iam/policy"
    29  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/iam/policymember"
    30  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/reconciliationinterval"
    31  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/tf"
    32  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/crd/crdgeneration"
    33  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/dcl/conversion"
    34  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/dcl/metadata"
    35  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/dcl/schema/dclschemaloader"
    36  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/k8s"
    37  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/servicemapping/servicemappingloader"
    38  	testcontroller "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/controller"
    39  	testk8s "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/k8s"
    40  	testservicemappingloader "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/servicemappingloader"
    41  
    42  	mmdcl "github.com/GoogleCloudPlatform/declarative-resource-client-library/dcl"
    43  	tfschema "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
    44  	"golang.org/x/sync/semaphore"
    45  	apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    46  	"k8s.io/apimachinery/pkg/api/errors"
    47  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    48  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    49  	"sigs.k8s.io/controller-runtime/pkg/event"
    50  	"sigs.k8s.io/controller-runtime/pkg/manager"
    51  	"sigs.k8s.io/controller-runtime/pkg/reconcile"
    52  )
    53  
    54  type ResourceCleanupPolicy string
    55  
    56  const (
    57  	// Always clean up resources.
    58  	CleanupPolicyAlways ResourceCleanupPolicy = "Always"
    59  	// Clean up resources on test success or while a test is successful, once the test enters a FAILed state do not
    60  	// clean up any more resources.
    61  	CleanupPolicyOnSuccess ResourceCleanupPolicy = "OnSuccess"
    62  )
    63  
    64  var (
    65  	ExpectedSuccessfulReconcileResultFor = expectedSuccessfulReconcileResultFor
    66  	ExpectedUnsuccessfulReconcileResult  = reconcile.Result{Requeue: false, RequeueAfter: 0 * time.Minute}
    67  	ExpectedRequeueReconcileStruct       = reconcile.Result{Requeue: true}
    68  )
    69  
    70  type TestReconciler struct {
    71  	mgr          manager.Manager
    72  	t            *testing.T
    73  	provider     *tfschema.Provider
    74  	smLoader     *servicemappingloader.ServiceMappingLoader
    75  	dclConfig    *mmdcl.Config
    76  	dclConverter *conversion.Converter
    77  }
    78  
    79  // TODO(kcc-eng): consolidate New() and NewForDCLAndTFTestReconciler() and keep the name as New() by refactoring all existing usages
    80  func New(t *testing.T, mgr manager.Manager, provider *tfschema.Provider) *TestReconciler {
    81  	return NewForDCLAndTFTestReconciler(t, mgr, provider, nil)
    82  }
    83  
    84  func NewForDCLAndTFTestReconciler(t *testing.T, mgr manager.Manager, provider *tfschema.Provider, dclConfig *mmdcl.Config) *TestReconciler {
    85  	smLoader := testservicemappingloader.New(t)
    86  	dclSchemaLoader, err := dclschemaloader.New()
    87  	if err != nil {
    88  		log.Fatalf("error creating a DCL schema loader: %v", err)
    89  	}
    90  	serviceMetaLoader := metadata.New()
    91  	dclConverter := conversion.New(dclSchemaLoader, serviceMetaLoader)
    92  	return &TestReconciler{
    93  		mgr:          mgr,
    94  		t:            t,
    95  		provider:     provider,
    96  		smLoader:     smLoader,
    97  		dclConverter: dclConverter,
    98  		dclConfig:    dclConfig,
    99  	}
   100  }
   101  
   102  func (r *TestReconciler) ReconcileIfManagedByKCC(unstruct *unstructured.Unstructured, expectedResult reconcile.Result, expectedErrorRegexp *regexp.Regexp) {
   103  	if k8s.IsManagedByKCC(unstruct.GroupVersionKind()) {
   104  		r.Reconcile(unstruct, expectedResult, expectedErrorRegexp)
   105  	} else {
   106  		// Some objects like Secrets should not be reconciled since they are
   107  		// not managed by KCC.
   108  		log.Printf("%v %v/%v is not managed by KCC; skipping reconciliation",
   109  			unstruct.GetKind(), unstruct.GetNamespace(), unstruct.GetName())
   110  	}
   111  }
   112  
   113  func (r *TestReconciler) Reconcile(unstruct *unstructured.Unstructured, expectedResult reconcile.Result, expectedErrorRegex *regexp.Regexp) {
   114  	r.t.Helper()
   115  	om := metav1.ObjectMeta{
   116  		Name:      unstruct.GetName(),
   117  		Namespace: unstruct.GetNamespace(),
   118  	}
   119  	r.ReconcileObjectMeta(om, unstruct.GetKind(), expectedResult, expectedErrorRegex)
   120  }
   121  
   122  func (r *TestReconciler) ReconcileObjectMeta(om metav1.ObjectMeta, kind string, expectedResult reconcile.Result, expectedErrorRegex *regexp.Regexp) {
   123  	r.t.Helper()
   124  	reconciler := r.NewReconcilerForKind(kind)
   125  	testcontroller.RunReconcilerAssertResults(r.t, reconciler, om, expectedResult, expectedErrorRegex)
   126  }
   127  
   128  // Creates and reconciles all unstructureds in the unstruct list. Returns a cleanup function that should be defered immediately after calling this function.
   129  func (r *TestReconciler) CreateAndReconcile(unstructs []*unstructured.Unstructured, cleanupPolicy ResourceCleanupPolicy) func() {
   130  	r.t.Helper()
   131  	cleanupFuncs := make([]func(), 0, len(unstructs))
   132  	for _, u := range unstructs {
   133  		if err := r.mgr.GetClient().Create(context.TODO(), u); err != nil {
   134  			r.t.Fatalf("error creating resource '%v': %v", u.GetKind(), err)
   135  		}
   136  		cleanupFuncs = append(cleanupFuncs, r.BuildCleanupFunc(u, cleanupPolicy))
   137  		r.ReconcileIfManagedByKCC(u, ExpectedSuccessfulReconcileResultFor(r, u), nil)
   138  	}
   139  	return func() {
   140  		for i := len(cleanupFuncs) - 1; i >= 0; i-- {
   141  			cleanupFuncs[i]()
   142  		}
   143  	}
   144  }
   145  
   146  func (r *TestReconciler) BuildCleanupFunc(unstruct *unstructured.Unstructured, cleanupPolicy ResourceCleanupPolicy) func() {
   147  	r.t.Helper()
   148  	return func() {
   149  		switch cleanupPolicy {
   150  		case CleanupPolicyAlways:
   151  			break
   152  		case CleanupPolicyOnSuccess:
   153  			if r.t.Failed() {
   154  				log.Printf("skipping cleanup of %v: %v/%v\n", unstruct.GetKind(), unstruct.GetNamespace(), unstruct.GetName())
   155  				return
   156  			}
   157  		default:
   158  			panic(fmt.Errorf("unknown cleanup policy: %v", cleanupPolicy))
   159  		}
   160  		log.Printf("Deleting %v: %v/%v\n", unstruct.GetKind(), unstruct.GetNamespace(), unstruct.GetName())
   161  		testk8s.RemoveDeletionDefenderFinalizerForUnstructured(r.t, unstruct, r.mgr.GetClient())
   162  		err := r.mgr.GetClient().Delete(context.TODO(), unstruct)
   163  		if err != nil {
   164  			if errors.IsNotFound(err) {
   165  				log.Printf("Resource already gone; no deletion required.")
   166  				return
   167  			}
   168  			r.t.Errorf("error deleting %v: %v", unstruct, err)
   169  		}
   170  		r.ReconcileIfManagedByKCC(unstruct, ExpectedSuccessfulReconcileResultFor(r, unstruct), nil)
   171  	}
   172  }
   173  
   174  func (r *TestReconciler) NewReconcilerForKind(kind string) reconcile.Reconciler {
   175  	r.t.Helper()
   176  	var reconciler reconcile.Reconciler
   177  	var err error
   178  	// Set 'immediateReconcileRequests' and 'resourceWatcherRoutines'
   179  	// to nil to disable reconciler's ability to create asynchronous
   180  	// watches on unready dependencies. This feature of the reconciler
   181  	// is unnecessary for our tests since we reconcile each dependency
   182  	// first before the resource under test is reconciled. Overall,
   183  	// the feature adds risk of complications due to it's multi-threaded
   184  	// nature.
   185  	var immediateReconcileRequests chan event.GenericEvent = nil
   186  	var resourceWatcherRoutines *semaphore.Weighted = nil
   187  
   188  	switch kind {
   189  	case "IAMPolicy":
   190  		reconciler, err = policy.NewReconciler(r.mgr, r.provider, r.smLoader, r.dclConverter, r.dclConfig, immediateReconcileRequests, resourceWatcherRoutines)
   191  	case "IAMPartialPolicy":
   192  		reconciler, err = partialpolicy.NewReconciler(r.mgr, r.provider, r.smLoader, r.dclConverter, r.dclConfig, immediateReconcileRequests, resourceWatcherRoutines)
   193  	case "IAMPolicyMember":
   194  		reconciler, err = policymember.NewReconciler(r.mgr, r.provider, r.smLoader, r.dclConverter, r.dclConfig, immediateReconcileRequests, resourceWatcherRoutines)
   195  	case "IAMAuditConfig":
   196  		reconciler, err = auditconfig.NewReconciler(r.mgr, r.provider, r.smLoader, r.dclConverter, r.dclConfig, immediateReconcileRequests, resourceWatcherRoutines)
   197  	default:
   198  		crd := testcontroller.GetCRDForKind(r.t, r.mgr.GetClient(), kind)
   199  		reconciler, err = r.newReconcilerForCRD(crd)
   200  	}
   201  	if err != nil {
   202  		r.t.Fatalf("error creating reconciler: %v", err)
   203  	}
   204  	return reconciler
   205  }
   206  
   207  func (r *TestReconciler) newReconcilerForCRD(crd *apiextensions.CustomResourceDefinition) (reconcile.Reconciler, error) {
   208  	if crd.GetLabels()[crdgeneration.ManagedByKCCLabel] == "true" {
   209  		// Set 'immediateReconcileRequests' and 'resourceWatcherRoutines'
   210  		// to nil to disable reconciler's ability to create asynchronous
   211  		// watches on unready dependencies. This feature of the reconciler
   212  		// is unnecessary for our tests since we reconcile each dependency
   213  		// first before the resource under test is reconciled. Overall,
   214  		// the feature adds risk of complications due to it's multi-threaded
   215  		// nature.
   216  		var immediateReconcileRequests chan event.GenericEvent = nil
   217  		var resourceWatcherRoutines *semaphore.Weighted = nil
   218  
   219  		if crd.GetLabels()[crdgeneration.TF2CRDLabel] == "true" {
   220  			return tf.NewReconciler(r.mgr, crd, r.provider, r.smLoader, immediateReconcileRequests, resourceWatcherRoutines)
   221  		}
   222  		if crd.GetLabels()[k8s.DCL2CRDLabel] == "true" {
   223  			return dclcontroller.NewReconciler(r.mgr, crd, r.dclConverter, r.dclConfig, r.smLoader, immediateReconcileRequests, resourceWatcherRoutines)
   224  		}
   225  	}
   226  	return nil, fmt.Errorf("CRD format not recognized")
   227  }
   228  
   229  func expectedSuccessfulReconcileResultFor(r *TestReconciler, u *unstructured.Unstructured) reconcile.Result {
   230  	if val, ok := k8s.GetAnnotation(k8s.ReconcileIntervalInSecondsAnnotation, u); ok {
   231  		reconcileInterval, err := reconciliationinterval.MeanReconcileReenqueuePeriodFromAnnotation(val)
   232  		if err != nil {
   233  			return reconcile.Result{}
   234  		}
   235  		return reconcile.Result{RequeueAfter: reconcileInterval}
   236  	}
   237  	return reconcile.Result{RequeueAfter: reconciliationinterval.MeanReconcileReenqueuePeriod(u.GroupVersionKind(), r.smLoader, r.dclConverter.MetadataLoader)}
   238  }
   239
View as plain text