dynamic_controller_integration_test.go

Documentation: github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/dynamic

     1  // Copyright 2022 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  //go:build integration
    16  // +build integration
    17  
    18  package dynamic_test
    19  
    20  import (
    21  	"context"
    22  	"flag"
    23  	"fmt"
    24  	"net/http"
    25  	"os"
    26  	"path/filepath"
    27  	"reflect"
    28  	"regexp"
    29  	"strings"
    30  	"testing"
    31  	"time"
    32  
    33  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/core/v1alpha1"
    34  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/dynamic"
    35  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/resourceactuation"
    36  	dclextension "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/dcl/extension"
    37  	dclmetadata "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/dcl/metadata"
    38  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/gcp"
    39  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/k8s"
    40  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/servicemapping/servicemappingloader"
    41  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test"
    42  	testcontroller "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/controller"
    43  	testreconciler "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/controller/reconciler"
    44  	testgcp "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/gcp"
    45  	testk8s "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/k8s"
    46  	testmain "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/main"
    47  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/resourcefixture"
    48  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/resourcefixture/contexts"
    49  	testrunner "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/runner"
    50  	testservicemapping "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/servicemapping"
    51  
    52  	"github.com/cenkalti/backoff"
    53  	"github.com/ghodss/yaml"
    54  	transport_tpg "github.com/hashicorp/terraform-provider-google-beta/google-beta/transport"
    55  	"k8s.io/apimachinery/pkg/api/errors"
    56  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    57  	"sigs.k8s.io/controller-runtime/pkg/client"
    58  	"sigs.k8s.io/controller-runtime/pkg/log"
    59  	"sigs.k8s.io/controller-runtime/pkg/manager"
    60  )
    61  
    62  type httpRoundTripperKeyType int
    63  
    64  // httpRoundTripperKey is the key value for http.RoundTripper in a context.Context
    65  var httpRoundTripperKey httpRoundTripperKeyType
    66  
    67  func init() {
    68  	// run-tests and skip-tests allows you to limit the tests that are run by
    69  	// specifying regexes to be used to match test names. See the
    70  	// formatTestName() function to see what test names look like.
    71  	flag.StringVar(&runTestsRegex, "run-tests", "", "run only the tests whose names match the given regex")
    72  	flag.StringVar(&skipTestsRegex, "skip-tests", "", "skip the tests whose names match the given regex, even those that match the run-tests regex")
    73  
    74  	// cleanup-resources allows you to disable the cleanup of resources created during testing. This can be useful for debugging test failures.
    75  	// The default value is true.
    76  	//
    77  	// To use this flag, you MUST use an equals sign as follows: go test -tags=integration -cleanup-resources=false
    78  	flag.BoolVar(&cleanupResources, "cleanup-resources", true, "when enabled, "+
    79  		"cloud resources created by tests will be cleaned up at the end of a test")
    80  
    81  	// Allow for capture of http requests during a test.
    82  	transport_tpg.DefaultHTTPClientTransformer = func(ctx context.Context, inner *http.Client) *http.Client {
    83  		ret := inner
    84  		if t := ctx.Value(httpRoundTripperKey); t != nil {
    85  			ret = &http.Client{Transport: t.(http.RoundTripper)}
    86  		}
    87  		if artifacts := os.Getenv("ARTIFACTS"); artifacts == "" {
    88  			log := log.FromContext(ctx)
    89  			log.Info("env var ARTIFACTS is not set; will not record http log")
    90  		} else {
    91  			outputDir := filepath.Join(artifacts, "http-logs")
    92  			t := test.NewHTTPRecorder(ret.Transport, outputDir)
    93  			ret = &http.Client{Transport: t}
    94  		}
    95  		return ret
    96  	}
    97  }
    98  
    99  var (
   100  	mgr              manager.Manager
   101  	runTestsRegex    string
   102  	skipTestsRegex   string
   103  	cleanupResources bool
   104  )
   105  
   106  const resourceIDTestVar = "${resourceId}"
   107  
   108  func shouldRunBasedOnRunAndSkipRegexes(parentTestName string, fixture resourcefixture.ResourceFixture) bool {
   109  	testName := formatTestName(parentTestName, fixture)
   110  
   111  	// If a skip-tests regex has been provided and it matches the test name, skip the test.
   112  	if skipTestsRegex != "" {
   113  		if regexp.MustCompile(skipTestsRegex).MatchString(testName) {
   114  			return false
   115  		}
   116  	}
   117  
   118  	// If a run-tests regex has been provided and it doesn't match the test name, skip the test.
   119  	if runTestsRegex != "" {
   120  		if !regexp.MustCompile(runTestsRegex).MatchString(testName) {
   121  			return false
   122  		}
   123  	}
   124  
   125  	return true
   126  }
   127  
   128  func TestAcquire(t *testing.T) {
   129  	t.Parallel()
   130  	shouldRun := func(fixture resourcefixture.ResourceFixture, mgr manager.Manager) bool {
   131  		if !shouldRunBasedOnRunAndSkipRegexes("TestAcquire", fixture) {
   132  			return false
   133  		}
   134  
   135  		// Never run the acquire test for 'containerannotations' test cases.
   136  		// This is because these test cases intentionally omit required
   137  		// hierarchical references to test that the webhooks default them
   138  		// correctly. However, the acquire test needs to be able to create GCP
   139  		// resources using create.yaml's without applying them onto the K8s API
   140  		// server, which means trying to create GCP resources using
   141  		// create.yaml's that are missing the required references and without
   142  		// going through the defaulting provided by webhooks.
   143  		if fixture.Type == resourcefixture.ContainerAnnotations {
   144  			return false
   145  		}
   146  
   147  		// Run the acquire test for 'resourceid' test cases to test that
   148  		// resources can be acquired using `spec.resourceID`.
   149  		if fixture.Type == resourcefixture.ResourceID {
   150  			return true
   151  		}
   152  
   153  		// Run the acquire test for a representative set of resource kinds.
   154  		// Note: ensuring that all fields are accounted for and not changed
   155  		// when applying the same YAMLs is handled separately by the NoChange
   156  		// test.
   157  		// TODO(b/239876828): Add "DataflowJob" back to acquisition tests.
   158  		kinds := map[string]bool{
   159  			// basic resource with no dependencies
   160  			"PubSubTopic": true,
   161  			// resource with dependencies
   162  			"PubSubSubscription": true,
   163  			// resource with no labels support
   164  			"BigQueryTable": true,
   165  			// resource acquirable by displayName and parent org/folder ID if
   166  			// server-generated ID (i.e. folder ID) is not specified
   167  			"Folder": true,
   168  			// used as an integration test verifying that falsey values are not
   169  			// incorrectly defaulted. (b/178744782)
   170  			"ComputeNetwork": true,
   171  		}
   172  		return kinds[fixture.GVK.Kind]
   173  	}
   174  	testFunc := func(t *testing.T, testContext testrunner.TestContext, systemContext testrunner.SystemContext) {
   175  		context := contexts.GetResourceContext(testContext.ResourceFixture, systemContext.DCLConverter.MetadataLoader, systemContext.DCLConverter.SchemaLoader)
   176  		testReconcileAcquire(t, testContext, systemContext, context)
   177  	}
   178  	testrunner.RunAllWithDependenciesCreatedButNotObject(t, mgr, shouldRun, testFunc)
   179  }
   180  
   181  func TestCreateNoChangeUpdateDelete(t *testing.T) {
   182  	t.Parallel()
   183  	shouldRun := func(fixture resourcefixture.ResourceFixture, mgr manager.Manager) bool {
   184  		switch fixture.Type {
   185  		case resourcefixture.IAMExternalOnlyRef, resourcefixture.IAMMemberReferences:
   186  			return false
   187  		}
   188  
   189  		// Skip ResourceID test cases for resources with server-generated IDs.
   190  		// These test cases exist solely to test acquisition of resources with
   191  		// server-generated IDs.
   192  		if fixture.Type == resourcefixture.ResourceID {
   193  			switch fixture.Name {
   194  			case "servergeneratedresourceid":
   195  				return false
   196  			case "servergeneratedresourceidfordcl":
   197  				return false
   198  			}
   199  		}
   200  
   201  		return shouldRunBasedOnRunAndSkipRegexes("TestCreateNoChangeUpdateDelete", fixture)
   202  	}
   203  	testFunc := func(t *testing.T, testContext testrunner.TestContext, systemContext testrunner.SystemContext) {
   204  		context := contexts.GetResourceContext(testContext.ResourceFixture, systemContext.DCLConverter.MetadataLoader, systemContext.DCLConverter.SchemaLoader)
   205  		testReconcileCreateNoChangeUpdateDelete(t, testContext, systemContext, context)
   206  	}
   207  	testrunner.RunAllWithDependenciesCreatedButNotObject(t, mgr, shouldRun, testFunc)
   208  }
   209  
   210  func formatTestName(parentTestName string, fixture resourcefixture.ResourceFixture) string {
   211  	return fmt.Sprintf("%v/%v", parentTestName, resourcefixture.FormatTestName(fixture))
   212  }
   213  func testCreate(t *testing.T, testContext testrunner.TestContext, systemContext testrunner.SystemContext, resourceContext contexts.ResourceContext) {
   214  	kubeClient := systemContext.Manager.GetClient()
   215  	initialUnstruct := testContext.CreateUnstruct.DeepCopy()
   216  	if err := kubeClient.Create(context.TODO(), initialUnstruct); err != nil {
   217  		t.Fatalf("error creating %v resource %v: %v", initialUnstruct.GetKind(), initialUnstruct.GetName(), err)
   218  	}
   219  	t.Logf("resource created with %v\r", initialUnstruct)
   220  	systemContext.Reconciler.Reconcile(initialUnstruct, testreconciler.ExpectedSuccessfulReconcileResultFor(systemContext.Reconciler, initialUnstruct), nil)
   221  	validateCreate(t, testContext, systemContext, resourceContext, initialUnstruct.GetGeneration())
   222  }
   223  
   224  func validateCreate(t *testing.T, testContext testrunner.TestContext, systemContext testrunner.SystemContext,
   225  	resourceContext contexts.ResourceContext, preReconcileGeneration int64) {
   226  	kubeClient := systemContext.Manager.GetClient()
   227  	initialUnstruct := testContext.CreateUnstruct.DeepCopy()
   228  	// Check labels match on create
   229  	reconciledUnstruct := &unstructured.Unstructured{
   230  		Object: map[string]interface{}{
   231  			"kind":       initialUnstruct.GetKind(),
   232  			"apiVersion": initialUnstruct.GetAPIVersion(),
   233  		},
   234  	}
   235  	if err := kubeClient.Get(context.TODO(), testContext.NamespacedName, reconciledUnstruct); err != nil {
   236  		t.Fatalf("unexpected error getting k8s resource: %v", err)
   237  	}
   238  	gcpUnstruct, err := resourceContext.Get(t, reconciledUnstruct, systemContext.TFProvider, kubeClient, systemContext.SMLoader, systemContext.DCLConfig, systemContext.DCLConverter)
   239  	if err != nil {
   240  		t.Fatalf("unexpected error when GETting '%v': %v", initialUnstruct.GetName(), err)
   241  	}
   242  	t.Logf("created resource is %v\r", gcpUnstruct)
   243  	if resourceContext.SupportsLabels(systemContext.SMLoader) {
   244  		testcontroller.AssertLabelsMatchAndHaveManagedLabel(t, gcpUnstruct.GetLabels(), reconciledUnstruct.GetLabels())
   245  	}
   246  
   247  	// Check that an "Updating" event was recorded, indicating that the
   248  	// controller tried to update the resource at all.
   249  	testcontroller.AssertEventRecordedforUnstruct(t, kubeClient, reconciledUnstruct, k8s.Updating)
   250  
   251  	// Check that condition is ready and "UpToDate" event was recorded
   252  	// TODO: (eventually) check default fields are propagated correctly
   253  	conditions := dynamic.GetConditions(t, reconciledUnstruct)
   254  	testcontroller.AssertReadyCondition(t, conditions)
   255  	testcontroller.AssertEventRecordedforUnstruct(t, kubeClient, reconciledUnstruct, k8s.UpToDate)
   256  
   257  	verifyResourceIDIfSupported(t, systemContext, resourceContext, reconciledUnstruct, initialUnstruct)
   258  
   259  	generationIncrease := reconciledUnstruct.GetGeneration() - preReconcileGeneration
   260  	if generationIncrease > 1 {
   261  		// Generation should have incremented at most once, only due to defaulted field sync-back.
   262  		t.Fatalf("unexpected generation increase %v", generationIncrease)
   263  	}
   264  
   265  	// Check observedGeneration matches with the pre-reconcile generation
   266  	testcontroller.AssertObservedGenerationEquals(t, reconciledUnstruct, preReconcileGeneration)
   267  }
   268  
   269  // testNoChange verifies that reconciling a resource which has not changed does not result in
   270  // any meaningful changes.
   271  func testNoChange(t *testing.T, testContext testrunner.TestContext, systemContext testrunner.SystemContext, resourceContext contexts.ResourceContext) {
   272  	if resourceContext.SkipNoChange {
   273  		return
   274  	}
   275  	kubeClient := systemContext.Manager.GetClient()
   276  	initialUnstruct := testContext.CreateUnstruct.DeepCopy()
   277  	if err := kubeClient.Get(context.TODO(), testContext.NamespacedName, initialUnstruct); err != nil {
   278  		t.Fatalf("unexpected error getting k8s resource: %v", err)
   279  	}
   280  	preReconcileGeneration := initialUnstruct.GetGeneration()
   281  
   282  	// Delete all events for the resource so that we can check later at the end
   283  	// of this test that the right events are recorded.
   284  	testcontroller.DeleteAllEventsForUnstruct(t, kubeClient, initialUnstruct)
   285  
   286  	// Reconcile resource without changing anything in its configuration
   287  	reconciledUnstruct := &unstructured.Unstructured{
   288  		Object: map[string]interface{}{
   289  			"kind":       initialUnstruct.GetKind(),
   290  			"apiVersion": initialUnstruct.GetAPIVersion(),
   291  		},
   292  	}
   293  	systemContext.Reconciler.Reconcile(initialUnstruct, testreconciler.ExpectedSuccessfulReconcileResultFor(systemContext.Reconciler, initialUnstruct), nil)
   294  	if err := kubeClient.Get(context.TODO(), testContext.NamespacedName, reconciledUnstruct); err != nil {
   295  		t.Fatalf("unexpected error getting k8s resource: %v", err)
   296  	}
   297  
   298  	if reconciledUnstruct.GetGeneration() != initialUnstruct.GetGeneration() {
   299  		t.Errorf("generation incremented during expected no-op")
   300  	}
   301  
   302  	if testContext.ResourceFixture.GVK.Kind == "DataflowJob" {
   303  		// Check that the Dataflow job has not been updated. This means
   304  		// checking that its status.jobId is still the same since Dataflow jobs
   305  		// are updated by creating a new job (which would have a new job ID) to
   306  		// replace the existing one.
   307  		checkDataflowJobNoChange(t, initialUnstruct, reconciledUnstruct)
   308  	}
   309  
   310  	// Check that an "Updating" event was never recorded.
   311  	testcontroller.AssertEventNotRecordedforUnstruct(t, kubeClient, initialUnstruct, k8s.Updating)
   312  
   313  	// Check observedGeneration matches with the pre-reconcile generation
   314  	testcontroller.AssertObservedGenerationEquals(t, reconciledUnstruct, preReconcileGeneration)
   315  }
   316  
   317  func testUpdate(t *testing.T, testContext testrunner.TestContext, systemContext testrunner.SystemContext, resourceContext contexts.ResourceContext) {
   318  	// Tests with `SkipUpdate` explicitly set to 'true' or tests for
   319  	// auto-generated resources don't support update test.
   320  	if resourceContext.SkipUpdate || resourceContext.IsAutoGenerated(systemContext.SMLoader) {
   321  		return
   322  	}
   323  	kubeClient := systemContext.Manager.GetClient()
   324  	initialUnstruct := testContext.CreateUnstruct.DeepCopy()
   325  	if err := kubeClient.Get(context.TODO(), testContext.NamespacedName, initialUnstruct); err != nil {
   326  		t.Fatalf("unexpected error getting k8s resource: %v", err)
   327  	}
   328  
   329  	// Delete all events for the resource so that we can check later at the end
   330  	// of this test that the right events are recorded.
   331  	testcontroller.DeleteAllEventsForUnstruct(t, kubeClient, initialUnstruct)
   332  
   333  	// Update resource from test data
   334  	updateUnstruct := testContext.UpdateUnstruct.DeepCopy()
   335  	if updateUnstruct == nil {
   336  		t.Fatalf("updateUnstruct is nil for '%v'. should SkipUpdate be set to true in resourcefixture/contexts?", testContext.ResourceFixture.Name)
   337  	}
   338  	updateUnstruct.SetResourceVersion(initialUnstruct.GetResourceVersion())
   339  	// For resources with server-generated IDs, ensure the relevant fields are in the status
   340  	status := initialUnstruct.Object["status"]
   341  	if err := unstructured.SetNestedField(updateUnstruct.Object, status, "status"); err != nil {
   342  		t.Fatalf("error setting status on updateUnstruct: %v", err)
   343  	}
   344  	patch := client.MergeFrom(testContext.CreateUnstruct)
   345  	t.Logf("patching %v with %v\r", updateUnstruct, patch)
   346  	if err := kubeClient.Patch(context.TODO(), updateUnstruct, patch); err != nil {
   347  		t.Fatalf("unexpected error when updating '%v': %v", initialUnstruct.GetName(), err)
   348  	}
   349  	preReconcileGeneration := updateUnstruct.GetGeneration()
   350  	systemContext.Reconciler.Reconcile(updateUnstruct, testreconciler.ExpectedSuccessfulReconcileResultFor(systemContext.Reconciler, updateUnstruct), nil)
   351  
   352  	reconciledUnstruct := &unstructured.Unstructured{
   353  		Object: map[string]interface{}{
   354  			"kind":       updateUnstruct.GetKind(),
   355  			"apiVersion": updateUnstruct.GetAPIVersion(),
   356  		},
   357  	}
   358  	if err := kubeClient.Get(context.TODO(), testContext.NamespacedName, reconciledUnstruct); err != nil {
   359  		t.Fatalf("unexpected error getting k8s resource: %v", err)
   360  	}
   361  
   362  	generationIncrease := reconciledUnstruct.GetGeneration() - updateUnstruct.GetGeneration()
   363  	if generationIncrease > 1 {
   364  		// Generation should have incremented at most once, only due to defaulted field sync-back.
   365  		t.Fatalf("unexpected generation increase %v", generationIncrease)
   366  	}
   367  
   368  	// Check labels match on update
   369  	gcpUnstruct, err := resourceContext.Get(t, reconciledUnstruct, systemContext.TFProvider, kubeClient, systemContext.SMLoader, systemContext.DCLConfig, systemContext.DCLConverter)
   370  	if err != nil {
   371  		t.Fatalf("unexpected error when GETting '%v': %v", updateUnstruct.GetName(), err)
   372  	}
   373  	if resourceContext.SupportsLabels(systemContext.SMLoader) {
   374  		testcontroller.AssertLabelsMatchAndHaveManagedLabel(t, gcpUnstruct.GetLabels(), testContext.UpdateUnstruct.GetLabels())
   375  	}
   376  
   377  	// If the object has a spec, check that the live GCP resource reflects the
   378  	// updates made by the update struct
   379  	if initialUnstruct.Object["spec"] != nil {
   380  		if gcpUnstruct.Object["spec"] == nil {
   381  			t.Fatalf("GCP resource has a nil spec even though it was created using a resource with a non-nil spec")
   382  		}
   383  		changedFields := getChangedFields(initialUnstruct.Object, reconciledUnstruct.Object, "spec")
   384  		assertObjectContains(t, gcpUnstruct.Object["spec"].(map[string]interface{}), changedFields)
   385  	}
   386  
   387  	// Check that an "Updating" event was recorded, indicating that the
   388  	// controller tried to update the resource at all.
   389  	testcontroller.AssertEventRecordedforUnstruct(t, kubeClient, reconciledUnstruct, k8s.Updating)
   390  
   391  	// Check if condition is ready and update event was recorded
   392  	conditions := dynamic.GetConditions(t, reconciledUnstruct)
   393  	testcontroller.AssertReadyCondition(t, conditions)
   394  	testcontroller.AssertEventRecordedforUnstruct(t, kubeClient, reconciledUnstruct, k8s.UpToDate)
   395  
   396  	// Check observedGeneration matches with the pre-reconcile generation
   397  	testcontroller.AssertObservedGenerationEquals(t, reconciledUnstruct, preReconcileGeneration)
   398  
   399  	verifyResourceIDIfSupported(t, systemContext, resourceContext, reconciledUnstruct, updateUnstruct)
   400  }
   401  
   402  // this test deletes the resource directly on GCP and then reconciles and verifies the resource was recreated correctly
   403  func testDriftCorrection(t *testing.T, testContext testrunner.TestContext, systemContext testrunner.SystemContext, resourceContext contexts.ResourceContext) {
   404  	if shouldSkipDriftDetection(t, resourceContext, systemContext.SMLoader, systemContext.DCLConverter.MetadataLoader, testContext.CreateUnstruct) {
   405  		return
   406  	}
   407  	kubeClient := systemContext.Manager.GetClient()
   408  	testUnstruct := testContext.CreateUnstruct.DeepCopy()
   409  	if err := kubeClient.Get(context.TODO(), testContext.NamespacedName, testUnstruct); err != nil {
   410  		t.Fatalf("unexpected error getting k8s resource: %v", err)
   411  	}
   412  	// For test cases with `cnrm.cloud.google.com/reconcile-interval-in-seconds` annotation set to 0, we should skip drift correction test.
   413  	if skip, _ := resourceactuation.ShouldSkip(testUnstruct); skip {
   414  		return
   415  	}
   416  	// Delete all events for the resource so that we can check later at the end
   417  	// of this test that the right events are recorded.
   418  	testcontroller.DeleteAllEventsForUnstruct(t, kubeClient, testUnstruct)
   419  
   420  	if err := resourceContext.Delete(t, testUnstruct, systemContext.TFProvider, systemContext.Manager.GetClient(), systemContext.SMLoader, systemContext.DCLConfig, systemContext.DCLConverter); err != nil {
   421  		t.Fatalf("error deleting: %v", err)
   422  	}
   423  	// Underlying APIs may not have strongly-consistent reads due to caching. Sleep before attempting a re-reconcile, to
   424  	// give the underlying system some time to propagate the deletion info.
   425  	time.Sleep(time.Second * 10)
   426  
   427  	// get the current state
   428  	t.Logf("reconcile with %v\r", testUnstruct)
   429  	systemContext.Reconciler.Reconcile(testUnstruct, testreconciler.ExpectedSuccessfulReconcileResultFor(systemContext.Reconciler, testUnstruct), nil)
   430  	t.Logf("reconciled with %v\r", testUnstruct)
   431  	validateCreate(t, testContext, systemContext, resourceContext, testUnstruct.GetGeneration())
   432  }
   433  
   434  func shouldSkipDriftDetection(t *testing.T, resourceContext contexts.ResourceContext, smLoader *servicemappingloader.ServiceMappingLoader,
   435  	serviceMetadataLoader dclmetadata.ServiceMetadataLoader, u *unstructured.Unstructured) bool {
   436  	if !testgcp.ResourceSupportsDeletion(u.GetKind()) {
   437  		// The drift correction test relies on being able to delete the underlying resource.
   438  		return true
   439  	}
   440  	if resourceContext.SkipDriftDetection {
   441  		return true
   442  	}
   443  
   444  	// Skip drift detection test for dcl-based resources with server-generated id.
   445  	if resourceContext.DCLBased {
   446  		s, found := dclextension.GetNameFieldSchema(resourceContext.DCLSchema)
   447  		if !found {
   448  			// The resource doesn't have a 'resourceID' field.
   449  			return false
   450  		}
   451  		isServerGenerated, err := dclextension.IsResourceIDFieldServerGenerated(s)
   452  		if err != nil {
   453  			t.Fatalf("error parsing `resourceID` field schema: %v", err)
   454  		}
   455  		return isServerGenerated
   456  	}
   457  	// Skip drift detection test for tf-based resources with server-generated id.
   458  	rc := testservicemapping.GetResourceConfig(t, smLoader, u)
   459  	return hasServerGeneratedId(*rc)
   460  }
   461  
   462  func hasServerGeneratedId(rc v1alpha1.ResourceConfig) bool {
   463  	return rc.ServerGeneratedIDField != ""
   464  }
   465  
   466  func testDelete(t *testing.T, testContext testrunner.TestContext, systemContext testrunner.SystemContext, resourceContext contexts.ResourceContext) {
   467  	if resourceContext.SkipDelete {
   468  		return
   469  	}
   470  	kubeClient := systemContext.Manager.GetClient()
   471  	testReconciler := systemContext.Reconciler
   472  	initialUnstruct := testContext.CreateUnstruct.DeepCopy()
   473  	if err := kubeClient.Delete(context.TODO(), initialUnstruct); err != nil {
   474  		t.Fatalf("error deleting resource: %v", err)
   475  	}
   476  
   477  	// Test that the deletion defender finalizer causes the resource to requeue
   478  	// and still exist on the underlying API
   479  	reconciledUnstruct := testContext.CreateUnstruct.DeepCopy()
   480  	testReconciler.Reconcile(reconciledUnstruct, testreconciler.ExpectedRequeueReconcileStruct, nil)
   481  	if err := kubeClient.Get(context.TODO(), testContext.NamespacedName, reconciledUnstruct); err != nil {
   482  		t.Fatalf("unexpected error getting k8s resource: %v", err)
   483  	}
   484  	if _, err := resourceContext.Get(t, reconciledUnstruct, systemContext.TFProvider, kubeClient, systemContext.SMLoader, systemContext.DCLConfig, systemContext.DCLConverter); err != nil {
   485  		t.Errorf("expected resource %s to not be deleted with deletion defender finalizer, but got error: %s",
   486  			initialUnstruct.GetName(), err)
   487  	}
   488  
   489  	// Perform the deletion on the underlying API
   490  	testk8s.RemoveDeletionDefenderFinalizerForUnstructured(t, reconciledUnstruct, kubeClient)
   491  	testReconciler.Reconcile(reconciledUnstruct, testreconciler.ExpectedSuccessfulReconcileResultFor(systemContext.Reconciler, reconciledUnstruct), nil)
   492  
   493  	if !testgcp.ResourceSupportsDeletion(testContext.ResourceFixture.GVK.Kind) {
   494  		_, err := resourceContext.Get(t, reconciledUnstruct, systemContext.TFProvider, kubeClient, systemContext.SMLoader, systemContext.DCLConfig, systemContext.DCLConverter)
   495  		if err != nil {
   496  			t.Errorf("expected resource %s to exist after deletion, but got error: %s", initialUnstruct.GetName(), err)
   497  		}
   498  	} else {
   499  		getFunc := func() error {
   500  			// for some resources, Get after Delete is eventually consistent, for that reason we retry until an error is returned
   501  			_, err := resourceContext.Get(t, reconciledUnstruct, systemContext.TFProvider, kubeClient, systemContext.SMLoader, systemContext.DCLConfig, systemContext.DCLConverter)
   502  			if err == nil {
   503  				return fmt.Errorf("expected error, instead got 'nil'")
   504  			}
   505  			return backoff.Permanent(err)
   506  		}
   507  		expBackoff := backoff.NewExponentialBackOff()
   508  		expBackoff.MaxElapsedTime = 60 * time.Second
   509  		expBackoff.MaxInterval = 10 * time.Second
   510  		err := backoff.Retry(getFunc, expBackoff)
   511  		// TODO: remove gcp.IsNotFoundError(...) once all resources are converted to use terraform for ResourceContext Create / Get
   512  		if !gcp.IsNotFoundError(err) && !contexts.IsNotFoundError(err) {
   513  			t.Errorf("expected GCP client to return NotFound for '%v', instead got: %v", initialUnstruct.GetName(), err)
   514  		}
   515  
   516  		err = kubeClient.Get(context.TODO(), testContext.NamespacedName, initialUnstruct)
   517  		if err == nil || !errors.IsNotFound(err) {
   518  			t.Errorf("unexpected error value: '%v'", err)
   519  		}
   520  	}
   521  
   522  	// Check that "Deleted" event was recorded
   523  	testcontroller.AssertEventRecordedforUnstruct(t, kubeClient, initialUnstruct, k8s.Deleted)
   524  }
   525  
   526  func testReconcileCreateNoChangeUpdateDelete(t *testing.T, testContext testrunner.TestContext, systemContext testrunner.SystemContext, resourceContext contexts.ResourceContext) {
   527  	resourceCleanup := systemContext.Reconciler.BuildCleanupFunc(testContext.CreateUnstruct, getResourceCleanupPolicy())
   528  	defer resourceCleanup()
   529  	testCreate(t, testContext, systemContext, resourceContext)
   530  	testNoChange(t, testContext, systemContext, resourceContext)
   531  	testUpdate(t, testContext, systemContext, resourceContext)
   532  	testDriftCorrection(t, testContext, systemContext, resourceContext)
   533  	testDelete(t, testContext, systemContext, resourceContext)
   534  }
   535  
   536  func checkComputeNetworkUpdate(t *testing.T, updateUnstruct *unstructured.Unstructured, gcpUnstruct *unstructured.Unstructured) {
   537  	expect, _, err := unstructured.NestedString(updateUnstruct.Object, "spec", "routingMode")
   538  	if err != nil {
   539  		t.Error(err)
   540  	}
   541  	actual, _, err := unstructured.NestedString(gcpUnstruct.Object, "routingConfig", "routingMode")
   542  	if err != nil {
   543  		t.Error(err)
   544  	}
   545  	if expect != actual {
   546  		t.Errorf("unexpected value for routingMode: got %v, want %v", actual, expect)
   547  	}
   548  }
   549  
   550  func checkDataflowJobNoChange(t *testing.T, initialUnstruct, reconciledUnstruct *unstructured.Unstructured) {
   551  	expectJobID, found, err := unstructured.NestedString(initialUnstruct.Object, "status", "jobId")
   552  	if err != nil {
   553  		t.Error(err)
   554  	}
   555  	if !found {
   556  		t.Errorf("initial unstruct does not have a status.jobId field")
   557  	}
   558  	actualJobID, found, err := unstructured.NestedString(reconciledUnstruct.Object, "status", "jobId")
   559  	if err != nil {
   560  		t.Error(err)
   561  	}
   562  	if !found {
   563  		t.Errorf("reconciled unstruct does not have a status.jobId field")
   564  	}
   565  	if expectJobID != actualJobID {
   566  		t.Errorf("expected status.jobId to be unchanged: got %v, want %v", actualJobID, expectJobID)
   567  	}
   568  }
   569  
   570  func testReconcileAcquire(t *testing.T, testContext testrunner.TestContext, systemContext testrunner.SystemContext, resourceContext contexts.ResourceContext) {
   571  	kubeClient := systemContext.Manager.GetClient()
   572  	initialUnstruct := testContext.CreateUnstruct.DeepCopy()
   573  
   574  	// Create the resource on GCP if it doesn't exist.
   575  	//
   576  	// If the unstruct contains a ${resourceId} test variable, that means its
   577  	// spec.resourceID is only meant to be used for acquisition. Therefore,
   578  	// strip out spec.resourceID for creation.
   579  	unstructToCreate := initialUnstruct.DeepCopy()
   580  	if containsResourceIDTestVar(t, unstructToCreate) {
   581  		testcontroller.RemoveResourceID(unstructToCreate)
   582  	}
   583  	var gcpUnstruct *unstructured.Unstructured
   584  	var err error
   585  	gcpUnstruct, err = resourceContext.Get(t, unstructToCreate, systemContext.TFProvider, kubeClient, systemContext.SMLoader, systemContext.DCLConfig, systemContext.DCLConverter)
   586  	if err != nil {
   587  		if !strings.Contains(err.Error(), "not found") {
   588  			t.Fatalf("unexpected error when GETting '%v': %v", unstructToCreate.GetName(), err)
   589  		}
   590  		if gcpUnstruct, err = resourceContext.Create(t, unstructToCreate, systemContext.TFProvider, kubeClient, systemContext.SMLoader, systemContext.DCLConfig, systemContext.DCLConverter); err != nil {
   591  			t.Fatalf("unexpected error when creating GCP resource '%v': %v", unstructToCreate.GetName(), err)
   592  		}
   593  	}
   594  
   595  	// Acquire the resource using the original unstruct.
   596  	//
   597  	// If the unstruct contains a ${resourceId} test variable, that means its
   598  	// spec.resourceID needs to be set with the live resource's resource ID to
   599  	// to make the unstruct usable for acquiring the live resource.
   600  	if containsResourceIDTestVar(t, initialUnstruct) {
   601  		resourceID, ok := testcontroller.GetResourceID(t, gcpUnstruct)
   602  		if !ok || resourceID == "" {
   603  			t.Fatalf("GCP resource does not have a %v field", k8s.ResourceIDFieldPath)
   604  		}
   605  		testcontroller.SetResourceID(t, initialUnstruct, resourceID)
   606  	}
   607  	// autoCreateSubnetworks defaults to true, rather than false, and acts
   608  	// as an end-to-end regression test for previous behavior defaulting to the wrong value
   609  	// see: b/178744782
   610  	if testContext.ResourceFixture.GVK.Kind == "ComputeNetwork" {
   611  		unstructured.RemoveNestedField(initialUnstruct.Object, "spec", "autoCreateSubnetworks")
   612  	}
   613  	if err := kubeClient.Create(context.TODO(), initialUnstruct); err != nil {
   614  		t.Fatalf("error creating resource: %v", err)
   615  	}
   616  	preReconcileGeneration := initialUnstruct.GetGeneration()
   617  	resourceCleanup := systemContext.Reconciler.BuildCleanupFunc(initialUnstruct, getResourceCleanupPolicy())
   618  	defer resourceCleanup()
   619  	systemContext.Reconciler.Reconcile(initialUnstruct, testreconciler.ExpectedSuccessfulReconcileResultFor(systemContext.Reconciler, initialUnstruct), nil)
   620  
   621  	// Check labels match
   622  	if resourceContext.SupportsLabels(systemContext.SMLoader) {
   623  		gcpUnstruct, err := resourceContext.Get(t, initialUnstruct, systemContext.TFProvider, kubeClient, systemContext.SMLoader, systemContext.DCLConfig, systemContext.DCLConverter)
   624  		if err != nil {
   625  			t.Fatalf("unexpected error when GETting '%v': %v", initialUnstruct.GetName(), err)
   626  		}
   627  		testcontroller.AssertLabelsMatchAndHaveManagedLabel(t, gcpUnstruct.GetLabels(), initialUnstruct.GetLabels())
   628  	}
   629  
   630  	reconciledUnstruct := &unstructured.Unstructured{
   631  		Object: map[string]interface{}{
   632  			"kind":       initialUnstruct.GetKind(),
   633  			"apiVersion": initialUnstruct.GetAPIVersion(),
   634  		},
   635  	}
   636  	if err := kubeClient.Get(context.TODO(), testContext.NamespacedName, reconciledUnstruct); err != nil {
   637  		t.Fatalf("unexpected error getting k8s resource: %v", err)
   638  	}
   639  
   640  	// Check that condition is ready and "UpToDate" event was recorded
   641  	conditions := dynamic.GetConditions(t, reconciledUnstruct)
   642  	testcontroller.AssertReadyCondition(t, conditions)
   643  	testcontroller.AssertEventRecordedforUnstruct(t, kubeClient, reconciledUnstruct, k8s.UpToDate)
   644  
   645  	// Check observedGeneration matches with the pre-reconcile generation
   646  	testcontroller.AssertObservedGenerationEquals(t, reconciledUnstruct, preReconcileGeneration)
   647  
   648  	verifyResourceIDIfSupported(t, systemContext, resourceContext, reconciledUnstruct, initialUnstruct)
   649  }
   650  
   651  // TODO(b/174100391): Compare the resourceID of the retrieved GCP resource and the appliedUnstruct.
   652  func verifyResourceIDIfSupported(t *testing.T, systemContext testrunner.SystemContext, resourceContext contexts.ResourceContext, reconciledUnstruct, appliedUnstruct *unstructured.Unstructured) {
   653  	if resourceContext.DCLBased {
   654  		s, found := dclextension.GetNameFieldSchema(resourceContext.DCLSchema)
   655  		if !found {
   656  			// The resource doesn't have a 'resourceID' field.
   657  			return
   658  		}
   659  		isServerGeneratedID, err := dclextension.IsResourceIDFieldServerGenerated(s)
   660  		if err != nil {
   661  			t.Fatalf("error parsing `resourceID` field schema: %v", err)
   662  		}
   663  		verifyResourceID(t, isServerGeneratedID, reconciledUnstruct, appliedUnstruct)
   664  	} else {
   665  		rc, err := systemContext.SMLoader.GetResourceConfig(reconciledUnstruct)
   666  		if err != nil {
   667  			t.Fatalf("error getting resource config for Kind '%s', "+
   668  				"Namespace '%s', Name '%s'", reconciledUnstruct.GetKind(),
   669  				reconciledUnstruct.GetNamespace(), reconciledUnstruct.GetName())
   670  		}
   671  		if !testcontroller.SupportsResourceIDField(rc) {
   672  			return
   673  		}
   674  		isServerGeneratedId := testcontroller.IsResourceIDFieldServerGenerated(rc)
   675  		verifyResourceID(t, isServerGeneratedId, reconciledUnstruct, appliedUnstruct)
   676  	}
   677  }
   678  
   679  func verifyResourceID(t *testing.T, isServerGeneratedID bool, reconciledUnstruct, appliedUnstruct *unstructured.Unstructured) {
   680  	reconciledResourceID, found := testcontroller.GetResourceID(t, reconciledUnstruct)
   681  	if !found {
   682  		t.Fatalf("'%s' not found", k8s.ResourceIDFieldPath)
   683  	}
   684  	if reconciledResourceID == "" {
   685  		t.Fatalf("invalid value for '%s': empty string",
   686  			k8s.ResourceIDFieldPath)
   687  	}
   688  
   689  	if isServerGeneratedID {
   690  		testcontroller.AssertServerGeneratedResourceIDMatch(t, reconciledResourceID, appliedUnstruct)
   691  		return
   692  	}
   693  	testcontroller.AssertUserSpecifiedResourceIDMatch(t, reconciledResourceID, appliedUnstruct)
   694  }
   695  
   696  func getResourceCleanupPolicy() testreconciler.ResourceCleanupPolicy {
   697  	if cleanupResources {
   698  		return testreconciler.CleanupPolicyAlways
   699  	}
   700  	return testreconciler.CleanupPolicyOnSuccess
   701  }
   702  
   703  func assertObjectContains(t *testing.T, obj, changedFields map[string]interface{}) {
   704  	for changedKey, changedVal := range changedFields {
   705  		objVal, ok := obj[changedKey]
   706  		if !ok {
   707  			t.Fatalf("object is missing the field %v", changedKey)
   708  		}
   709  
   710  		switch changedVal.(type) {
   711  		case map[string]interface{}:
   712  			if _, ok := objVal.(map[string]interface{}); !ok {
   713  				t.Fatalf("expected object to have a map at %v, but got %v", changedKey, objVal)
   714  			}
   715  			assertObjectContains(t, objVal.(map[string]interface{}), changedVal.(map[string]interface{}))
   716  		default:
   717  			if !reflect.DeepEqual(objVal, changedVal) {
   718  				t.Fatalf("unexpected value for %v: got %v, want %v", changedKey, objVal, changedVal)
   719  			}
   720  		}
   721  	}
   722  }
   723  
   724  func getChangedFields(initialObject, updatedObject map[string]interface{}, field string) map[string]interface{} {
   725  	changedFields := make(map[string]interface{})
   726  	initial, _ := initialObject[field].(map[string]interface{})
   727  	updated := updatedObject[field].(map[string]interface{})
   728  	if !reflect.DeepEqual(initial, updated) {
   729  		for k, v := range updated {
   730  			if !reflect.DeepEqual(initial[k], v) {
   731  				if _, ok := v.(map[string]interface{}); ok {
   732  					changedFields[k] = getChangedFields(initial, updated, k)
   733  				} else {
   734  					changedFields[k] = updated[k]
   735  				}
   736  			}
   737  		}
   738  	}
   739  	return changedFields
   740  }
   741  
   742  // stripInternalKeysFromManagedFields strips fields that are used for internal
   743  // tracking of managed fields by the api-server. This is useful when trying to
   744  // differentiate between changes made by the server vs changed made by our controllers.
   745  //
   746  // note: this will modify the struct permanently, and will make the schema for
   747  // managedFields invalid. Do not use this method without a deepcopy if you are
   748  // planning on extracting the ManagedField struct later.
   749  func stripInternalKeysFromManagedFields(t *testing.T, unstruct *unstructured.Unstructured) {
   750  	managedFields, found, err := unstructured.NestedFieldNoCopy(unstruct.Object, "metadata", "managedFields")
   751  	if err != nil {
   752  		t.Fatalf("error getting managed fields: %v", err)
   753  	}
   754  	if !found {
   755  		return
   756  	}
   757  	for _, managedField := range managedFields.([]interface{}) {
   758  		managedField := managedField.(map[string]interface{})
   759  		delete(managedField, "time")
   760  	}
   761  }
   762  
   763  func containsResourceIDTestVar(t *testing.T, u *unstructured.Unstructured) bool {
   764  	b, err := yaml.Marshal(u)
   765  	if err != nil {
   766  		t.Fatalf("error marshalling unstruct to bytes: %v", err)
   767  	}
   768  	return strings.Contains(string(b), resourceIDTestVar)
   769  }
   770  
   771  func TestMain(m *testing.M) {
   772  	testmain.TestMainForIntegrationTests(m, &mgr)
   773  }
   774
View as plain text