samples.go

Documentation: github.com/GoogleCloudPlatform/k8s-config-connector/config/tests/samples/create

     1  // Copyright 2022 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package create
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"io/ioutil"
    21  	"path"
    22  	"regexp"
    23  	"sort"
    24  	"strconv"
    25  	"strings"
    26  	"sync"
    27  	"testing"
    28  	"time"
    29  
    30  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/dynamic"
    31  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/k8s"
    32  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test"
    33  	testcontroller "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/controller"
    34  	testgcp "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/gcp"
    35  	testvariable "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/resourcefixture/variable"
    36  	testyaml "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/yaml"
    37  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/util/repo"
    38  
    39  	"github.com/ghodss/yaml"
    40  	"github.com/golang-collections/go-datastructures/queue"
    41  	"k8s.io/apimachinery/pkg/api/errors"
    42  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    43  	"k8s.io/apimachinery/pkg/util/wait"
    44  	"sigs.k8s.io/controller-runtime/pkg/log"
    45  )
    46  
    47  type Sample struct {
    48  	Name      string
    49  	Resources []*unstructured.Unstructured
    50  }
    51  
    52  func sortSamplesInDescendingOrderByNumberOfResources(samples []Sample) {
    53  	sort.Slice(samples, func(i, j int) bool {
    54  		return len(samples[i].Resources) > len(samples[j].Resources)
    55  	})
    56  }
    57  
    58  func networksInSampleCount(sample Sample) int {
    59  	count := 0
    60  	for _, r := range sample.Resources {
    61  		if r.GetKind() == "ComputeNetwork" {
    62  			count += 1
    63  		}
    64  	}
    65  	return count
    66  }
    67  
    68  func SetupNamespacesAndApplyDefaults(t *Harness, samples []Sample, project testgcp.GCPProject) {
    69  	namespaceNames := getNamespaces(samples)
    70  	setupNamespaces(t, namespaceNames, project)
    71  }
    72  
    73  func setupNamespaces(t *Harness, namespaces []string, project testgcp.GCPProject) {
    74  	for _, n := range namespaces {
    75  		testcontroller.SetupNamespaceForProject(t.T, t.GetClient(), n, project.ProjectID)
    76  	}
    77  }
    78  
    79  func getNamespaces(samples []Sample) []string {
    80  	namespaces := make(map[string]bool)
    81  	for _, sample := range samples {
    82  		for _, unstruct := range sample.Resources {
    83  			namespaces[unstruct.GetNamespace()] = true
    84  		}
    85  	}
    86  	results := make([]string, 0, len(namespaces))
    87  	for k := range namespaces {
    88  		results = append(results, k)
    89  	}
    90  	return results
    91  }
    92  
    93  func RunCreateDeleteTest(t *Harness, unstructs []*unstructured.Unstructured, cleanupResources bool) {
    94  	// Create and reconcile all resources & dependencies
    95  	for _, u := range unstructs {
    96  		if err := t.GetClient().Create(context.TODO(), u); err != nil {
    97  			t.Fatalf("error creating resource: %v", err)
    98  		}
    99  	}
   100  	waitForReady(t, unstructs)
   101  	// Clean up resources on success or if cleanupResources flag is true
   102  	if cleanupResources {
   103  		cleanup(t, unstructs)
   104  	}
   105  }
   106  
   107  func waitForReady(t *Harness, unstructs []*unstructured.Unstructured) {
   108  	var wg sync.WaitGroup
   109  	for _, u := range unstructs {
   110  		wg.Add(1)
   111  		go waitForReadySingleResource(t, &wg, u)
   112  	}
   113  	wg.Wait()
   114  }
   115  
   116  func waitForReadySingleResource(t *Harness, wg *sync.WaitGroup, u *unstructured.Unstructured) {
   117  	logger := log.FromContext(t.Ctx)
   118  
   119  	name := k8s.GetNamespacedName(u)
   120  	defer wg.Done()
   121  	err := wait.PollImmediate(15*time.Second, 35*time.Minute, func() (done bool, err error) {
   122  		done = true
   123  		logger.Info("Testing to see if resource is ready", "kind", u.GetKind(), "name", u.GetName())
   124  		err = t.GetClient().Get(t.Ctx, name, u)
   125  		if err != nil {
   126  			logger.Info("Error getting resource", "kind", u.GetKind(), "name", u.GetName(), "error", err)
   127  			return false, nil
   128  		}
   129  		if u.GetKind() == "Secret" { // If unstruct is a Secret and it is found on the API server, then the Secret is ready
   130  			return true, nil
   131  		}
   132  		if u.Object["status"] == nil ||
   133  			u.Object["status"].(map[string]interface{})["conditions"] == nil { // status not ready
   134  			logger.Info("resource does not yet have status or conditions", "kind", u.GetKind(), "name", u.GetName())
   135  			return false, nil
   136  		}
   137  		conditions := dynamic.GetConditions(t.T, u)
   138  		for _, c := range conditions {
   139  			if c.Type == "Ready" && c.Status == "True" {
   140  				logger.Info("resource is ready", "kind", u.GetKind(), "name", u.GetName())
   141  				return true, nil
   142  			}
   143  		}
   144  		// This resource is not completely ready. Let's keep polling.
   145  		logger.Info("resource is not ready", "kind", u.GetKind(), "name", u.GetName(),
   146  			"conditions", conditions)
   147  		return false, nil
   148  	})
   149  	if err == nil {
   150  		return
   151  	}
   152  	if err != wait.ErrWaitTimeout {
   153  		t.Errorf("error while polling for ready on %v with name '%v': %v", u.GetKind(), u.GetName(), err)
   154  		return
   155  	}
   156  	baseMsg := fmt.Sprintf("timed out waiting for ready on %v with name '%v'", u.GetKind(), u.GetName())
   157  	if err := t.GetClient().Get(t.Ctx, name, u); err != nil {
   158  		t.Errorf("%v, error retrieving final status.conditions: %v", baseMsg, err)
   159  		return
   160  	}
   161  	conditions := dynamic.GetConditions(t.T, u)
   162  	if len(conditions) == 0 {
   163  		t.Errorf("%v, no conditions on resource", baseMsg)
   164  		return
   165  	}
   166  	t.Errorf("%v, final status.conditions: %v", baseMsg, conditions)
   167  }
   168  
   169  func cleanup(t *Harness, unstructs []*unstructured.Unstructured) {
   170  	logger := log.FromContext(t.Ctx)
   171  
   172  	for _, u := range unstructs {
   173  		logger.Info("Deleting resource", "kind", u.GetKind(), "name", u.GetName())
   174  		if err := t.GetClient().Delete(t.Ctx, u); err != nil {
   175  			t.Errorf("error deleting: %v", err)
   176  		}
   177  	}
   178  	var wg sync.WaitGroup
   179  	for _, u := range unstructs {
   180  		wg.Add(1)
   181  		go waitForDeleteToComplete(t, &wg, u)
   182  	}
   183  	wg.Wait()
   184  }
   185  
   186  func waitForDeleteToComplete(t *Harness, wg *sync.WaitGroup, u *unstructured.Unstructured) {
   187  	defer wg.Done()
   188  	// Do a best-faith cleanup of the resources. Gives a 30 minute buffer for cleanup, though
   189  	// resources that can be cleaned up quicker exit earlier.
   190  	err := wait.PollImmediate(15*time.Second, 30*time.Minute, func() (bool, error) {
   191  		if err := t.GetClient().Get(t.Ctx, k8s.GetNamespacedName(u), u); !errors.IsNotFound(err) {
   192  			return false, nil
   193  		}
   194  		return true, nil
   195  	})
   196  	// TODO (b/197783299): think of better way to handle resources that take a longer time to cleanup
   197  	if err != nil {
   198  		t.Errorf("error while polling for resource cleanup on %v with name '%v': %v; last seen status: %v", u.GetKind(), u.GetName(), err, u.Object["status"])
   199  	}
   200  }
   201  
   202  // LoadSamples loads all the samples
   203  func LoadSamples(t *testing.T, project testgcp.GCPProject) []Sample {
   204  	matchEverything := regexp.MustCompile(".*")
   205  	return loadSamplesOntoUnstructs(t, matchEverything, project)
   206  }
   207  
   208  func loadSamplesOntoUnstructs(t *testing.T, regex *regexp.Regexp, project testgcp.GCPProject) []Sample {
   209  	t.Helper()
   210  
   211  	samples := make([]Sample, 0)
   212  	sampleNamesToFiles := mapSampleNamesToFilePaths(t, regex)
   213  	subVars := newSubstitutionVariables(t, project)
   214  	for sample, files := range sampleNamesToFiles {
   215  		resources := make([]*unstructured.Unstructured, 0)
   216  		for _, f := range files {
   217  			unstructs := readFileToUnstructs(t, f, subVars)
   218  			resources = append(resources, unstructs...)
   219  		}
   220  		s := Sample{
   221  			Name:      sample,
   222  			Resources: resources,
   223  		}
   224  		samples = append(samples, s)
   225  	}
   226  	return samples
   227  }
   228  
   229  func mapSampleNamesToFilePaths(t *testing.T, regex *regexp.Regexp) map[string][]string {
   230  	t.Helper()
   231  	samples := make(map[string][]string)
   232  	q := queue.New(1)
   233  	q.Put(repo.GetResourcesSamplesPath())
   234  	for !q.Empty() {
   235  		items, err := q.Get(1)
   236  		if err != nil {
   237  			t.Fatalf("error retrieving an item from queue: %v", err)
   238  		}
   239  		dir := items[0].(string)
   240  		fileInfos, err := ioutil.ReadDir(dir)
   241  		if err != nil {
   242  			t.Fatalf("error reading directory '%v': %v", dir, err)
   243  		}
   244  		for _, fi := range fileInfos {
   245  			if fi.IsDir() {
   246  				q.Put(path.Join(dir, fi.Name()))
   247  				continue
   248  			}
   249  			if !strings.HasSuffix(fi.Name(), ".yaml") {
   250  				continue
   251  			}
   252  			sampleName := path.Base(dir)
   253  			if !regex.MatchString(sampleName) {
   254  				continue
   255  			}
   256  			filePath := path.Join(dir, fi.Name())
   257  			samples[sampleName] = append(samples[sampleName], filePath)
   258  		}
   259  	}
   260  	return samples
   261  }
   262  
   263  func newSubstitutionVariables(t *testing.T, project testgcp.GCPProject) map[string]string {
   264  	subs := make(map[string]string)
   265  	subs["${HOST_PROJECT_ID?}"] = project.ProjectID
   266  	subs["${PROJECT_ID?}"] = project.ProjectID
   267  	subs["${PROJECT_NUMBER?}"] = strconv.FormatInt(project.ProjectNumber, 10)
   268  	subs["${FOLDER_ID?}"] = testgcp.GetFolderID(t)
   269  	subs["${ORG_ID?}"] = testgcp.GetOrgID(t)
   270  	subs["${BILLING_ACCOUNT_ID?}"] = testgcp.GetBillingAccountID(t)
   271  	subs["${BILLING_ACCOUNT_ID_FOR_BILLING_RESOURCES?}"] = testgcp.GetTestBillingAccountIDForBillingResources(t)
   272  	subs["${GSA_EMAIL?}"] = getKCCServiceAccountEmail(t, project)
   273  	subs["${DLP_TEST_BUCKET?}"] = testgcp.GetDLPTestBucket(t)
   274  	return subs
   275  }
   276  
   277  // getKCCServiceAccountEmail attempts to get the email address of the service
   278  // account used by KCC.
   279  func getKCCServiceAccountEmail(t *testing.T, project testgcp.GCPProject) string {
   280  	// If there is a service account configured via "Application Default
   281  	// Credentials", then assume this is the service account used by KCC. This
   282  	// assumption holds true if the test is run by Prow.
   283  	if sa, err := testgcp.FindDefaultServiceAccount(); err != nil {
   284  		t.Fatalf("error from FindDefaultServiceAccount: %v", err)
   285  	} else if sa != "" {
   286  		return sa
   287  	}
   288  	// Otherwise, assume the project has a standard, cluster-mode KCC service
   289  	// account set up.
   290  	return fmt.Sprintf("cnrm-system@%v.iam.gserviceaccount.com", project.ProjectID)
   291  }
   292  
   293  func readFileToUnstructs(t *testing.T, fileName string, subVars map[string]string) []*unstructured.Unstructured {
   294  	t.Helper()
   295  	var returnUnstructs []*unstructured.Unstructured
   296  
   297  	b := testcontroller.ReadFileToBytes(t, fileName)
   298  	s := string(b)
   299  	for k, v := range subVars {
   300  		s = strings.ReplaceAll(s, k, v)
   301  	}
   302  	b = []byte(s)
   303  
   304  	yamls := testyaml.SplitYAML(t, b)
   305  	for _, b = range yamls {
   306  		u := test.ToUnstructWithNamespace(t, b, subVars["${PROJECT_ID?}"])
   307  		returnUnstructs = append(returnUnstructs, u)
   308  	}
   309  	return returnUnstructs
   310  }
   311  
   312  func replaceResourceNamesWithUniqueIDs(t *testing.T, unstructs []*unstructured.Unstructured) []*unstructured.Unstructured {
   313  	namesToBeReplaced := make([]string, 0)
   314  	for _, u := range unstructs {
   315  		namesToBeReplaced = append(namesToBeReplaced, u.GetName())
   316  	}
   317  
   318  	// Replace names in order of descending length to avoid collisions. For
   319  	// example, unstructs might have instances of names "resource-dep" and
   320  	// "resource-dep2". If we do a string replacement of "resource-dep" first,
   321  	// then the string "resource-dep2" will also be affected since it contains
   322  	// "resource-dep".
   323  	namesToBeReplaced = sortByDescendingLen(namesToBeReplaced)
   324  
   325  	namesToUniqueIDs := make(map[string]string)
   326  	idReg := regexp.MustCompile("[a-z]")
   327  	for _, n := range namesToBeReplaced {
   328  		namesToUniqueIDs[n] = testvariable.RandomIdGenerator(idReg, uint(len(n)))
   329  	}
   330  
   331  	newUnstructs := make([]*unstructured.Unstructured, 0)
   332  	for _, u := range unstructs {
   333  		b, err := yaml.Marshal(u)
   334  		if err != nil {
   335  			t.Fatalf("error marshalling unstruct to bytes: %v", err)
   336  		}
   337  		s := string(b)
   338  		for _, name := range namesToBeReplaced {
   339  			uniqueID := namesToUniqueIDs[name]
   340  			s = strings.ReplaceAll(s, name, uniqueID)
   341  		}
   342  		b = []byte(s)
   343  		newUnstruct := &unstructured.Unstructured{}
   344  		err = yaml.Unmarshal(b, newUnstruct)
   345  		if err != nil {
   346  			t.Fatalf("error unmarshalling bytes to unstruct: %v", err)
   347  		}
   348  		// Folders also need to have unique values for spec.displayName
   349  		if newUnstruct.GetKind() == "Folder" {
   350  			newDisplayName, err := generateNewFolderDisplayName(u, idReg)
   351  			if err != nil {
   352  				t.Fatalf("error generating new spec.displayName value for Folder '%v': %v", u.GetName(), err)
   353  			}
   354  			unstructured.SetNestedField(newUnstruct.Object, newDisplayName, "spec", "displayName")
   355  		}
   356  		newUnstructs = append(newUnstructs, newUnstruct)
   357  	}
   358  	return newUnstructs
   359  }
   360  
   361  // generateNewFolderDisplayName returns a string that can be used as a new
   362  // display name for the given Folder sample. It has the same length as the
   363  // original display name used in the sample, and it contains enough randomly
   364  // generated characters to avoid display name collisions.
   365  func generateNewFolderDisplayName(folderUnstruct *unstructured.Unstructured, idReg *regexp.Regexp) (string, error) {
   366  	newDisplayNamePrefix := "KCC "
   367  	uniqueIDLen := 10
   368  	minDisplayNameLen := len(newDisplayNamePrefix) + uniqueIDLen
   369  
   370  	displayName, err := getFolderDisplayName(folderUnstruct)
   371  	if err != nil {
   372  		return "", err
   373  	}
   374  
   375  	if len(displayName) < minDisplayNameLen {
   376  		return "", fmt.Errorf("Folder '%v' has a spec.displayName value of "+
   377  			"'%v' which is too short; please use a spec.displayName with at "+
   378  			"least '%v' characters", folderUnstruct.GetName(), displayName, minDisplayNameLen)
   379  	}
   380  
   381  	return newDisplayNamePrefix + testvariable.RandomIdGenerator(idReg, uint(len(displayName)-len(newDisplayNamePrefix))), nil
   382  }
   383  
   384  func getFolderDisplayName(folderUnstruct *unstructured.Unstructured) (string, error) {
   385  	displayName, ok, err := unstructured.NestedString(folderUnstruct.Object, "spec", "displayName")
   386  	if err != nil {
   387  		return "", fmt.Errorf("error getting spec.displayName of Folder unstruct: %v", err)
   388  	}
   389  	if !ok {
   390  		return "", fmt.Errorf("spec.displayName not found for Folder unstruct")
   391  	}
   392  	return displayName, nil
   393  }
   394  
   395  func sortByDescendingLen(strs []string) []string {
   396  	strsCopy := append(make([]string, 0), strs...)
   397  	sort.Slice(strsCopy, func(i, j int) bool {
   398  		return len(strsCopy[i]) > len(strsCopy[j])
   399  	})
   400  	return strsCopy
   401  }
   402
View as plain text