harness.go

Documentation: github.com/GoogleCloudPlatform/k8s-config-connector/config/tests/samples/create

     1  // Copyright 2022 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package create
    16  
    17  import (
    18  	"context"
    19  	"net/http"
    20  	"os"
    21  	"path/filepath"
    22  	"sync"
    23  	"testing"
    24  	"time"
    25  
    26  	"github.com/GoogleCloudPlatform/k8s-config-connector/mockgcp"
    27  	"github.com/GoogleCloudPlatform/k8s-config-connector/mockgcp/pkg/storage"
    28  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/dynamic"
    29  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/kccmanager"
    30  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/kccmanager/nocache"
    31  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/registration"
    32  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/crd/crdloader"
    33  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/logging"
    34  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test"
    35  	testenvironment "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/environment"
    36  	testwebhook "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/webhook"
    37  	cnrmwebhook "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/webhook"
    38  
    39  	transport_tpg "github.com/hashicorp/terraform-provider-google-beta/google-beta/transport"
    40  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    41  	"k8s.io/apimachinery/pkg/runtime/schema"
    42  	"k8s.io/apimachinery/pkg/types"
    43  	"k8s.io/apimachinery/pkg/util/wait"
    44  	"k8s.io/client-go/rest"
    45  	"sigs.k8s.io/controller-runtime/pkg/client"
    46  	"sigs.k8s.io/controller-runtime/pkg/envtest"
    47  	"sigs.k8s.io/controller-runtime/pkg/log"
    48  	"sigs.k8s.io/controller-runtime/pkg/manager"
    49  	"sigs.k8s.io/controller-runtime/pkg/webhook"
    50  )
    51  
    52  type Harness struct {
    53  	*testing.T
    54  	Ctx context.Context
    55  
    56  	client     client.Client
    57  	restConfig *rest.Config
    58  }
    59  
    60  type httpRoundTripperKeyType int
    61  
    62  // httpRoundTripperKey is the key value for http.RoundTripper in a context.Context
    63  var httpRoundTripperKey httpRoundTripperKeyType
    64  
    65  // NewHarnessWithManager builds a Harness for an existing manager.
    66  // deprecated: Prefer NewHarness, which can construct a manager and mock gcp etc.
    67  func NewHarnessWithManager(t *testing.T, ctx context.Context, mgr manager.Manager) *Harness {
    68  	h := &Harness{
    69  		T:      t,
    70  		Ctx:    ctx,
    71  		client: mgr.GetClient(),
    72  	}
    73  	return h
    74  }
    75  
    76  func NewHarness(t *testing.T, ctx context.Context) *Harness {
    77  	ctx, cancel := context.WithCancel(ctx)
    78  	t.Cleanup(func() {
    79  		cancel()
    80  	})
    81  
    82  	log := log.FromContext(ctx)
    83  
    84  	h := &Harness{
    85  		T:   t,
    86  		Ctx: ctx,
    87  	}
    88  
    89  	kccConfig := kccmanager.Config{}
    90  	// Prevent manager from binding to a port to serve prometheus metrics
    91  	// since creating multiple managers for tests will fail if more than
    92  	// one manager tries to bind to the same port.
    93  	kccConfig.ManagerOptions.MetricsBindAddress = "0"
    94  	// Prevent manager from binding to a port to serve health probes since
    95  	// creating multiple managers for tests will fail if more than one
    96  	// manager tries to bind to the same port.
    97  	kccConfig.ManagerOptions.HealthProbeBindAddress = "0"
    98  	// supply a concrete client to disable the default behavior of caching
    99  	kccConfig.ManagerOptions.NewClient = nocache.NoCacheClientFunc
   100  
   101  	var webhooks []cnrmwebhook.WebhookConfig
   102  
   103  	loadCRDs := true
   104  	if targetKube := os.Getenv("E2E_KUBE_TARGET"); targetKube == "envtest" {
   105  		whCfgs, err := testwebhook.GetTestCommonWebhookConfigs()
   106  		if err != nil {
   107  			h.Fatalf("error getting common wehbook configs: %v", err)
   108  		}
   109  		webhooks = append(webhooks, whCfgs...)
   110  
   111  		env := &envtest.Environment{
   112  			ControlPlaneStartTimeout: time.Minute,
   113  		}
   114  
   115  		testenvironment.ConfigureWebhookInstallOptions(env, whCfgs)
   116  
   117  		h.Logf("starting envtest apiserver")
   118  		restConfig, err := env.Start()
   119  		if err != nil {
   120  			h.Fatalf("error starting test environment: %v", err)
   121  		}
   122  
   123  		t.Cleanup(func() {
   124  			if err := env.Stop(); err != nil {
   125  				h.Errorf("error stopping envtest environment: %v", err)
   126  			}
   127  		})
   128  
   129  		h.restConfig = restConfig
   130  
   131  		kccConfig.ManagerOptions.Port = env.WebhookInstallOptions.LocalServingPort
   132  		kccConfig.ManagerOptions.Host = env.WebhookInstallOptions.LocalServingHost
   133  		kccConfig.ManagerOptions.CertDir = env.WebhookInstallOptions.LocalServingCertDir
   134  	} else {
   135  		t.Fatalf("E2E_KUBE_TARGET=%q not supported", targetKube)
   136  	}
   137  
   138  	if h.client == nil {
   139  		client, err := client.New(h.restConfig, client.Options{})
   140  		if err != nil {
   141  			h.Fatalf("error building client: %v", err)
   142  		}
   143  		h.client = client
   144  	}
   145  
   146  	logging.SetupLogger()
   147  
   148  	if loadCRDs {
   149  		crds, err := crdloader.LoadCRDs()
   150  		if err != nil {
   151  			h.Fatalf("error loading crds: %v", err)
   152  		}
   153  		{
   154  			var wg sync.WaitGroup
   155  			for i := range crds {
   156  				crd := &crds[i]
   157  				wg.Add(1)
   158  				t.Logf("loading crd %v", crd.GetName())
   159  				go func() {
   160  					defer wg.Done()
   161  					if err := h.client.Create(ctx, crd.DeepCopy()); err != nil {
   162  						h.Fatalf("error creating crd %v: %v", crd.GroupVersionKind(), err)
   163  					}
   164  					h.waitForCRDReady(crd)
   165  				}()
   166  			}
   167  			wg.Wait()
   168  		}
   169  	}
   170  
   171  	if targetGCP := os.Getenv("E2E_GCP_TARGET"); targetGCP == "mock" {
   172  		t.Logf("creating mock gcp")
   173  
   174  		mockCloud := mockgcp.NewMockRoundTripper(t, h.client, storage.NewInMemoryStorage())
   175  
   176  		roundTripper := http.RoundTripper(mockCloud)
   177  
   178  		h.Ctx = context.WithValue(h.Ctx, httpRoundTripperKey, roundTripper)
   179  
   180  		kccConfig.HTTPClient = &http.Client{Transport: roundTripper}
   181  
   182  		kccConfig.AccessToken = "dummytoken"
   183  	} else if targetGCP := os.Getenv("E2E_GCP_TARGET"); targetGCP == "real" {
   184  		t.Logf("targeting real GCP")
   185  	} else {
   186  		t.Fatalf("E2E_GCP_TARGET=%q not supported", targetGCP)
   187  	}
   188  
   189  	transport_tpg.DefaultHTTPClientTransformer = func(ctx context.Context, inner *http.Client) *http.Client {
   190  		ret := inner
   191  		if t := ctx.Value(httpRoundTripperKey); t != nil {
   192  			ret = &http.Client{Transport: t.(http.RoundTripper)}
   193  		}
   194  		if artifacts := os.Getenv("ARTIFACTS"); artifacts == "" {
   195  			log.Info("env var ARTIFACTS is not set; will not record http log")
   196  		} else {
   197  			outputDir := filepath.Join(artifacts, "http-logs")
   198  			t := test.NewHTTPRecorder(ret.Transport, outputDir)
   199  			ret = &http.Client{Transport: t}
   200  		}
   201  		return ret
   202  	}
   203  
   204  	transport_tpg.OAuth2HTTPClientTransformer = func(ctx context.Context, inner *http.Client) *http.Client {
   205  		ret := inner
   206  		if t := ctx.Value(httpRoundTripperKey); t != nil {
   207  			ret = &http.Client{Transport: t.(http.RoundTripper)}
   208  		}
   209  		if artifacts := os.Getenv("ARTIFACTS"); artifacts == "" {
   210  			log.Info("env var ARTIFACTS is not set; will not record http log")
   211  		} else {
   212  			outputDir := filepath.Join(artifacts, "http-logs")
   213  			t := test.NewHTTPRecorder(ret.Transport, outputDir)
   214  			ret = &http.Client{Transport: t}
   215  		}
   216  		return ret
   217  	}
   218  
   219  	mgr, err := kccmanager.New(h.Ctx, h.restConfig, kccConfig)
   220  	if err != nil {
   221  		t.Fatalf("error creating new manager: %v", err)
   222  	}
   223  	if len(webhooks) > 0 {
   224  		server := mgr.GetWebhookServer()
   225  		for _, cfg := range webhooks {
   226  			server.Register(cfg.Path, &webhook.Admission{Handler: cfg.Handler})
   227  		}
   228  	}
   229  
   230  	// Register the deletion defender controller.
   231  	if err := registration.Add(mgr, nil, nil, nil, nil, registration.RegisterDeletionDefenderController); err != nil {
   232  		t.Fatalf("error adding registration controller for deletion defender controllers: %v", err)
   233  	}
   234  	// Start the manager, Start(...) is a blocking operation so it needs to be done asynchronously.
   235  	errors := make(chan error)
   236  	go func() {
   237  		err := mgr.Start(ctx)
   238  		if err != nil {
   239  			t.Errorf("error from mgr.Start: %v", err)
   240  		}
   241  		errors <- err
   242  	}()
   243  
   244  	t.Cleanup(func() {
   245  		cancel() // because cleanups run last-in-first-out, we need to cancel again
   246  		if err := <-errors; err != nil {
   247  			t.Errorf("error from mgr.Start: %v", err)
   248  		}
   249  	})
   250  
   251  	return h
   252  }
   253  
   254  func (h *Harness) GetClient() client.Client {
   255  	return h.client
   256  }
   257  
   258  func MaybeSkip(t *testing.T, name string, resources []*unstructured.Unstructured) {
   259  	if os.Getenv("E2E_GCP_TARGET") == "mock" {
   260  		for _, resource := range resources {
   261  			gvk := resource.GroupVersionKind()
   262  			switch gvk.GroupKind() {
   263  			case schema.GroupKind{Group: "iam.cnrm.cloud.google.com", Kind: "IAMServiceAccount"}:
   264  				// ok
   265  
   266  			case schema.GroupKind{Group: "networkservices.cnrm.cloud.google.com", Kind: "NetworkServicesMesh"}:
   267  				// ok
   268  
   269  			case schema.GroupKind{Group: "privateca.cnrm.cloud.google.com", Kind: "PrivateCACAPool"}:
   270  				// ok
   271  
   272  			case schema.GroupKind{Group: "secretmanager.cnrm.cloud.google.com", Kind: "SecretManagerSecret"}:
   273  				// ok
   274  			case schema.GroupKind{Group: "secretmanager.cnrm.cloud.google.com", Kind: "SecretManagerSecretVersion"}:
   275  				// ok
   276  
   277  			case schema.GroupKind{Group: "", Kind: "Secret"}:
   278  				// ok
   279  
   280  			case schema.GroupKind{Group: "serviceusage.cnrm.cloud.google.com", Kind: "Service"}:
   281  			case schema.GroupKind{Group: "serviceusage.cnrm.cloud.google.com", Kind: "ServiceIdentity"}:
   282  				// ok
   283  
   284  			default:
   285  				t.Skipf("gk %v not suppported by mock gcp; skipping", gvk.GroupKind())
   286  			}
   287  		}
   288  	}
   289  }
   290  
   291  func (t *Harness) waitForCRDReady(obj client.Object) {
   292  	logger := log.FromContext(t.Ctx)
   293  
   294  	apiVersion, kind := obj.GetObjectKind().GroupVersionKind().ToAPIVersionAndKind()
   295  	name := obj.GetName()
   296  	namespace := obj.GetNamespace()
   297  
   298  	id := types.NamespacedName{Name: name, Namespace: namespace}
   299  	if err := wait.PollImmediate(2*time.Second, 1*time.Minute, func() (bool, error) {
   300  		u := &unstructured.Unstructured{}
   301  		u.SetAPIVersion(apiVersion)
   302  		u.SetKind(kind)
   303  		logger.Info("Testing to see if resource is ready", "kind", kind, "id", id)
   304  		if err := t.GetClient().Get(t.Ctx, id, u); err != nil {
   305  			logger.Info("Error getting resource", "kind", kind, "id", id, "error", err)
   306  			return false, err
   307  		}
   308  		conditions := dynamic.GetConditions(t.T, u)
   309  		for _, condition := range conditions {
   310  			if condition.Type == "Established" && condition.Status == "True" {
   311  				logger.Info("crd is ready", "kind", kind, "id", id)
   312  				return true, nil
   313  			}
   314  		}
   315  		// This resource is not completely ready. Let's keep polling.
   316  		logger.Info("CRD is not ready", "kind", kind, "id", id, "conditions", conditions)
   317  		return false, nil
   318  	}); err != nil {
   319  		t.Errorf("error while polling for ready on %v %v: %v", kind, id, err)
   320  		return
   321  	}
   322  }
   323
View as plain text