main.go

Documentation: github.com/GoogleCloudPlatform/k8s-config-connector/scripts/resource-autogen

     1  // Copyright 2023 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package main
    16  
    17  import (
    18  	"encoding/json"
    19  	"fmt"
    20  	"os"
    21  	"path/filepath"
    22  	"reflect"
    23  	"regexp"
    24  	"sort"
    25  	"strings"
    26  
    27  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/core/v1alpha1"
    28  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/k8s"
    29  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/krmtotf"
    30  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/servicemapping/servicemappingloader"
    31  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/text"
    32  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/util/fileutil"
    33  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/util/repo"
    34  	"github.com/GoogleCloudPlatform/k8s-config-connector/scripts/resource-autogen/allowlist"
    35  	"github.com/GoogleCloudPlatform/k8s-config-connector/scripts/resource-autogen/sampleconversion"
    36  	autogenloader "github.com/GoogleCloudPlatform/k8s-config-connector/scripts/resource-autogen/servicemapping/servicemappingloader"
    37  
    38  	"github.com/hashicorp/go-multierror"
    39  	"github.com/tmccombs/hcl2json/convert"
    40  	"k8s.io/apimachinery/pkg/runtime/schema"
    41  	"k8s.io/klog/v2"
    42  	"sigs.k8s.io/yaml"
    43  )
    44  
    45  var (
    46  	randomSuffixKeyword = "%{random_suffix}"
    47  	uniqueIDHolder      = "${uniqueId}"
    48  	// nonIDKRMFieldsRequiringUniqueValuesMap is the map of KRM kinds and the
    49  	// map of their non-ID string fields that require unique values.
    50  	nonIDKRMFieldsRequiringUniqueValuesMap = map[string]map[string]bool{
    51  		"TagsTagKey": {
    52  			"shortName": true,
    53  		},
    54  		"AccessContextManagerServicePerimeter": {
    55  			"title": true,
    56  		},
    57  	}
    58  	// krmFieldsNotAllowingSpecialCharsMap is the map of KRM kinds and the map
    59  	// of their string fields that don't allow special characters in the value.
    60  	// This is needed for the sample converter to explicitly clean up any
    61  	// special characters in the value.
    62  	krmFieldsNotAllowingSpecialCharsMap = map[string]map[string]bool{
    63  		"Project": {
    64  			"name": true,
    65  		},
    66  	}
    67  	// additionalRequiredFieldsMap is the map of KRM kinds and the maps of their
    68  	// required string fields and the default values that are not specified in
    69  	// the TF sample.
    70  	// This is needed so that the sample converter can add required fields and
    71  	// values.
    72  	additionalRequiredFieldsMap = map[string]map[string]string{
    73  		"DataCatalogTaxonomy": {
    74  			"region": "us",
    75  		},
    76  	}
    77  	// defaultOrganizationalResourcesMap is the map of organizational KRM kinds
    78  	// and the relative resource names of the default test instances.
    79  	// There are some organizational resources that KCC shouldn't touch in the
    80  	// integration test. When those resources are needed as the dependency,
    81  	// instead of creating new ones, we should use the default/pre-created ones
    82  	// instead.
    83  	defaultOrganizationalResourcesMap = map[string]string{
    84  		"AccessContextManagerAccessPolicy": "accessPolicies/578359180191",
    85  	}
    86  	// ListFieldsWithAtMostOneItemMap is the map of KRM kinds and the maps of
    87  	// their list fields that support at most one item.
    88  	// TF resources have many list fields that support at most one item, and KCC
    89  	// turns those list fields into object fields. Sample converter should be
    90  	// able to identify those fields and turn them from lists to objects in the
    91  	// converted YAML.
    92  	ListFieldsWithAtMostOneItemMap = map[string]map[string]bool{
    93  		"AccessContextManagerServicePerimeter": {
    94  			"status": true,
    95  		},
    96  	}
    97  	// ResourceIDLengthMap is the map of KRM kinds and the length limitation of
    98  	// their resource IDs.
    99  	// Some resources have more strict ID length limitations than others. KCC's
   100  	// test resourceID may not always fit. This map is used to track the lengths
   101  	// of resource IDs that may cause issues if not properly handled.
   102  	ResourceIDLengthMap = map[string]int{
   103  		"AccessContextManagerServicePerimeter": 50,
   104  	}
   105  )
   106  
   107  func main() {
   108  	if err := run(); err != nil {
   109  		fmt.Fprintf(os.Stderr, "%v\n", err)
   110  		os.Exit(1)
   111  	}
   112  }
   113  
   114  func run() error {
   115  	smLoader, err := servicemappingloader.New()
   116  	if err != nil {
   117  		return fmt.Errorf("error getting new service mapping loader: %w", err)
   118  	}
   119  	generatedSMMap, err := autogenloader.GetGeneratedSMMap()
   120  	if err != nil {
   121  		return fmt.Errorf("error getting the generated ServiceMapping map: %w", err)
   122  	}
   123  	autoGenAllowlist, err := allowlist.LoadAutoGenAllowList(generatedSMMap)
   124  	if err != nil {
   125  		return fmt.Errorf("error loading allowlist for autogen resources: %w", err)
   126  	}
   127  	tfToGVK, err := GetTFTypeToGVKMap(smLoader)
   128  	if err != nil {
   129  		return fmt.Errorf("error getting TF type mapping: %w", err)
   130  	}
   131  	err = convertTFSamplesToKRMTestdata(tfToGVK, smLoader, autoGenAllowlist)
   132  	if err != nil {
   133  		return fmt.Errorf("error converting TF samples:\n%w", err)
   134  	}
   135  	return nil
   136  }
   137  
   138  func convertTFSamplesToKRMTestdata(tfToGVK map[string]schema.GroupVersionKind, smLoader *servicemappingloader.ServiceMappingLoader, autoGenAllowlist *allowlist.AutoGenAllowlist) error {
   139  	var errs *multierror.Error
   140  	samplesPath := repo.GetAutoGeneratedTFSamplesPathOrFatal()
   141  	sampleFolders, err := fileutil.SubdirsIn(samplesPath)
   142  	if err != nil {
   143  		return fmt.Errorf("error reading directory %v: %w", samplesPath, err)
   144  	}
   145  	generatedSamples := make(map[string]bool)
   146  	for _, sf := range sampleFolders {
   147  		originalSF := sf
   148  		sf = strings.Replace(sf, "dry-run", "dry_run", -1)
   149  		klog.Infof("Converting TF sample %v (original name: %v)...", sf, originalSF)
   150  		sampleNameInfo := strings.Split(sf, "-")
   151  		if len(sampleNameInfo) < 3 || len(sampleNameInfo) > 4 {
   152  			errs = multierror.Append(errs,
   153  				fmt.Errorf("sample folder name should be in the format of '[Service]-[Kind]-[sample_name]' or '[Service]-[Kind]-[sample_name]-skipped', but it's %v", sf))
   154  			return errs
   155  		}
   156  		service := sampleNameInfo[0]
   157  		kind := sampleNameInfo[1]
   158  		group := fmt.Sprintf("%s.cnrm.cloud.google.com", strings.ToLower(service))
   159  
   160  		autoGenType, ok := autoGenAllowlist.GetKRMKind(kind)
   161  		if !ok {
   162  			klog.Infof("Skipping the parse of sample %v. Kind %v not allowlisted.", sf, kind)
   163  			continue
   164  		}
   165  		sm, err := smLoader.GetServiceMapping(group)
   166  		if err != nil {
   167  			// TODO(b/265225406): Check error type.
   168  			klog.Infof("Skipping the parse of sample %v. Group %v not found.", sf, group)
   169  			continue
   170  		}
   171  		rc := servicemappingloader.GetResourceConfigsForKind(sm, kind)
   172  		if rc == nil || len(rc) == 0 {
   173  			klog.Infof("Skipping the parse of sample %v. Kind %v not found in service mapping.", sf, kind)
   174  			continue
   175  		}
   176  		// Auto-generated resources should have one-on-one mapping between kind
   177  		// and resource configs.
   178  		if len(rc) > 1 {
   179  			errs = multierror.Append(errs,
   180  				fmt.Errorf("error retrieving resource configs for "+
   181  					"kind %v, there should only be one matching resource config", kind))
   182  			return errs
   183  		}
   184  		// If the TF type for the sample is not an auto-generated kind, no need
   185  		// to parse the sample.
   186  		if !rc[0].AutoGenerated {
   187  			klog.Infof("Skipping the parse of sample %v. Kind %v is not auto-generated.", sf, kind)
   188  			continue
   189  		}
   190  		// If it is an organizational resource that shouldn't be created, no
   191  		// need to parse the sample.
   192  		if _, ok := defaultOrganizationalResourcesMap[kind]; ok {
   193  			klog.Infof("Skipping the parse of sample %v. Creating a resource of kind %v will impact the use of the test organization.", sf, kind)
   194  			continue
   195  		}
   196  
   197  		sampleName := text.SnakeCaseToLowerCase(sampleNameInfo[2])
   198  		// Focus on basic samples for now.
   199  		if !strings.HasSuffix(sampleName, "basic") {
   200  			klog.Infof("Skipping the parse of sample %v. This is not a basic sample.", sf)
   201  			continue
   202  		}
   203  
   204  		path := filepath.Join(samplesPath, sf, "main.tf")
   205  		b, err := os.ReadFile(path)
   206  		if err != nil {
   207  			errToReturn := fmt.Errorf("error reading file %v for TF sample %s: %w", path, sf, err)
   208  			klog.Warningf("Failed sample conversion: %v", errToReturn)
   209  			errs = multierror.Append(errs, errToReturn)
   210  			continue
   211  		}
   212  
   213  		jsonStruct, err := convertHCLBytesToJSON(b)
   214  		if err != nil {
   215  			errToReturn := fmt.Errorf("error converting HCL to JSON for TF sample %s: %v", sf, err)
   216  			klog.Warningf("Failed sample conversion: %v", errToReturn)
   217  			errs = multierror.Append(errs, errToReturn)
   218  			continue
   219  		}
   220  
   221  		create, dependencies, err := tfSampleToKRMTestData(kind, jsonStruct, tfToGVK, smLoader)
   222  		if err != nil {
   223  			errToReturn := fmt.Errorf("error converting TF samples to KRM test data for TF sample %s: %v", sf, err)
   224  			klog.Warningf("Failed sample conversion: %v", errToReturn)
   225  			errs = multierror.Append(errs, errToReturn)
   226  			continue
   227  		}
   228  
   229  		if err := insertTestData(create, dependencies, autoGenType, sampleName, generatedSamples); err != nil {
   230  			errToReturn := fmt.Errorf("error unmarshaling json for TF sample %s: %v", sf, err)
   231  			klog.Warningf("Failed sample conversion: %v", errToReturn)
   232  			errs = multierror.Append(errs, errToReturn)
   233  			continue
   234  		}
   235  
   236  		klog.Infof("Sample %v converted successfully!", sf)
   237  	}
   238  
   239  	return errs.ErrorOrNil()
   240  }
   241  
   242  func convertHCLBytesToJSON(raw []byte) (map[string]interface{}, error) {
   243  	lines := strings.Split(string(raw), "\n")
   244  	hcl := ""
   245  	for _, s := range lines {
   246  		trimmed := strings.TrimSpace(s)
   247  		if len(trimmed) == 0 || trimmed == "```hcl" || trimmed == "```" {
   248  			continue
   249  		}
   250  		hcl += s + "\n"
   251  	}
   252  	hcl = strings.TrimSuffix(hcl, "\n")
   253  
   254  	// To bypass the "Invalid Template Control Keyword" error.
   255  	if strings.Contains(hcl, randomSuffixKeyword) {
   256  		hcl = strings.ReplaceAll(hcl, randomSuffixKeyword, uniqueIDHolder)
   257  	}
   258  
   259  	input := []byte(hcl)
   260  	convertedBytes, err := convert.Bytes(input, "", convert.Options{})
   261  	if err != nil {
   262  		return nil, fmt.Errorf("error parsing bytes: %v", err)
   263  	}
   264  
   265  	jsonStruct := make(map[string]interface{})
   266  	err = json.Unmarshal(convertedBytes, &jsonStruct)
   267  	if err != nil {
   268  		return nil, fmt.Errorf("error unmarshaling json: %v", err)
   269  	}
   270  
   271  	return jsonStruct, nil
   272  }
   273  
   274  func tfSampleToKRMTestData(testKind string, tf map[string]interface{}, tfToGVK map[string]schema.GroupVersionKind, smLoader *servicemappingloader.ServiceMappingLoader) (create map[string]interface{}, dependencies []map[string]interface{}, err error) {
   275  	resourcesRaw, ok := tf["resource"]
   276  	if !ok {
   277  		return nil, nil, fmt.Errorf("tf struct should contain a 'resource' field: %+v", tf)
   278  	}
   279  	resources, ok := resourcesRaw.(map[string]interface{})
   280  	if !ok {
   281  		return nil, nil, fmt.Errorf("value of 'resource' should be in the format of 'map[string]interface{}' but not %T", resourcesRaw)
   282  	}
   283  
   284  	create = make(map[string]interface{})
   285  	dependencyMap := make(map[string]map[string]interface{})
   286  	dependencyGraph := sampleconversion.NewDependencyGraph()
   287  	for tfType, resource := range resources {
   288  		gvk, ok := tfToGVK[tfType]
   289  		if !ok {
   290  			return nil, nil, fmt.Errorf("TF type %v doesn't exist in the service mappings", tfType)
   291  		}
   292  		// No need to parse the config for organizational resources that
   293  		// shouldn't be created.
   294  		if _, ok := defaultOrganizationalResourcesMap[gvk.Kind]; ok {
   295  			continue
   296  		}
   297  
   298  		sm, err := smLoader.GetServiceMapping(gvk.Group)
   299  		if err != nil {
   300  			return nil, nil, err
   301  		}
   302  		rc, err := servicemappingloader.GetResourceConfigsForTFType(sm, tfType)
   303  		if err != nil {
   304  			return nil, nil, err
   305  		}
   306  
   307  		krmConfig, err := tfConfigToKRMConfig(resource, tfType, dependencyGraph, *rc, tfToGVK)
   308  		if err != nil {
   309  			return nil, nil, fmt.Errorf("error converting TF config to KRM for resource\n%+v\n:\n%w", resource, err)
   310  		}
   311  		if gvk.Kind == testKind {
   312  			if len(create) != 0 {
   313  				return nil, nil, fmt.Errorf("more than one resource of type %s exists, but there should be only one", tfType)
   314  			}
   315  			create = krmConfig
   316  		} else {
   317  			dependencyMap[tfType] = krmConfig
   318  		}
   319  	}
   320  
   321  	sortedRefDependencyTypes := dependencyGraph.TopologicalSort()
   322  	dependencies = make([]map[string]interface{}, 0)
   323  	for _, tfType := range sortedRefDependencyTypes {
   324  		gvk, ok := tfToGVK[tfType]
   325  		if !ok {
   326  			return nil, nil, fmt.Errorf("TF type %v doesn't exist in the service mappings", tfType)
   327  		}
   328  		// The "create" struct is covered in the dependencyGraph when there is
   329  		// any dependency, but it shouldn't be added to "dependencies" struct
   330  		// list.
   331  		if gvk.Kind == testKind {
   332  			continue
   333  		}
   334  		dependency, ok := dependencyMap[tfType]
   335  		if !ok {
   336  			return nil, nil, fmt.Errorf("TF type %v doesn't exist in the dependencyMap", tfType)
   337  		}
   338  		dependencies = append(dependencies, dependency)
   339  		delete(dependencyMap, tfType)
   340  	}
   341  	// dependencyGraph only covers TF types that are involved in references. We
   342  	// still need to go through other configs whose TF types are not involved in
   343  	// any reference.
   344  	sortedNonRefDependencyTypes := make([]string, 0)
   345  	for tfType, _ := range dependencyMap {
   346  		sortedNonRefDependencyTypes = append(sortedNonRefDependencyTypes, tfType)
   347  	}
   348  	sort.Strings(sortedNonRefDependencyTypes)
   349  	for _, tfType := range sortedNonRefDependencyTypes {
   350  		dependency, ok := dependencyMap[tfType]
   351  		if !ok {
   352  			return nil, nil, fmt.Errorf("TF type %v doesn't exist in the dependencyMap", tfType)
   353  		}
   354  		dependencies = append(dependencies, dependency)
   355  	}
   356  	return create, dependencies, nil
   357  }
   358  
   359  func tfConfigToKRMConfig(tfConfig interface{}, tfType string, dependencyGraph *sampleconversion.DependencyGraph, rc v1alpha1.ResourceConfig, tfToGVK map[string]schema.GroupVersionKind) (spec map[string]interface{}, err error) {
   360  	klog.V(2).Infof("tfConfig: %+v\n", tfConfig)
   361  
   362  	name, specs, containerAnnotation, err := cleanupTFFields(tfConfig, tfType, dependencyGraph, rc, tfToGVK)
   363  	if err != nil {
   364  		return nil, fmt.Errorf("error cleanning up the TF config: %w", err)
   365  	}
   366  	// TODO(b/265367038): Handle the samples with multiple resources of the same type.\
   367  	gvk, ok := tfToGVK[tfType]
   368  	if !ok {
   369  		return nil, fmt.Errorf("TF type %v doesn't exist in the service mappings", tfType)
   370  	}
   371  	return handleKRMFields(specs[name], containerAnnotation, gvk, rc)
   372  }
   373  
   374  func cleanupTFFields(configRaw interface{}, tfType string, dependencyGraph *sampleconversion.DependencyGraph, rc v1alpha1.ResourceConfig, tfToGVK map[string]schema.GroupVersionKind) (name string, specs map[string]map[string]interface{}, containerAnnotation map[string]string, err error) {
   375  	config, ok := configRaw.(map[string]interface{})
   376  	if !ok {
   377  		return "", nil, nil, fmt.Errorf("TF config should be in the format of 'map[string]interface{}' but not %T", configRaw)
   378  	}
   379  	if len(config) != 1 {
   380  		return "", nil, nil, fmt.Errorf("there should be only 1 element, but got %v", len(config))
   381  	}
   382  
   383  	name = reflect.ValueOf(config).MapKeys()[0].String()
   384  	specRaw := config[name]
   385  	specArray, ok := specRaw.([]interface{})
   386  	if !ok {
   387  		return "", nil, nil, fmt.Errorf("value of '%s' should be in the format of '[]interface{}' but not %T", name, specRaw)
   388  	}
   389  	if len(specArray) != 1 {
   390  		return "", nil, nil, fmt.Errorf("there should be only 1 element, but got %v", len(specArray))
   391  	}
   392  	spec, ok := specArray[0].(map[string]interface{})
   393  	if !ok {
   394  		return "", nil, nil, fmt.Errorf("configuration value should be in the format of 'map[string]interface{}' but not %T", specArray[0])
   395  	}
   396  
   397  	provider, ok := spec["provider"]
   398  	if ok {
   399  		if provider != "${google-beta}" {
   400  			return "", nil, nil, fmt.Errorf("illegal provider value: %s", provider)
   401  		}
   402  		delete(spec, "provider")
   403  	}
   404  
   405  	_, ok = spec["lifecycle"]
   406  	if ok {
   407  		delete(spec, "lifecycle")
   408  	}
   409  
   410  	additionalRequiredFields, ok := additionalRequiredFieldsMap[rc.Kind]
   411  	if ok {
   412  		for f, v := range additionalRequiredFields {
   413  			if _, ok := spec[f]; !ok {
   414  				spec[f] = v
   415  			}
   416  		}
   417  	}
   418  
   419  	krmSpec, containerAnnotation, err := krmifySpec(spec, tfType, dependencyGraph, rc, tfToGVK)
   420  	if err != nil {
   421  		return "", nil, nil, fmt.Errorf("error krmifying the spec %+v: %w", spec, err)
   422  	}
   423  	specs = make(map[string]map[string]interface{})
   424  	specs[name] = krmSpec
   425  	return name, specs, containerAnnotation, nil
   426  }
   427  
   428  func krmifySpec(tfSpec map[string]interface{}, tfType string, dependencyGraph *sampleconversion.DependencyGraph, rc v1alpha1.ResourceConfig, tfToGVK map[string]schema.GroupVersionKind) (krmSpec map[string]interface{}, containerAnnotation map[string]string, err error) {
   429  	krmSpec = make(map[string]interface{})
   430  	containerAnnotation = make(map[string]string)
   431  	refConfigMap := getReferenceConfigMap(rc)
   432  	containerMap := getContainerMap(rc)
   433  	for tfFieldName, value := range tfSpec {
   434  		krmFieldName := text.SnakeCaseToLowerCamelCase(tfFieldName)
   435  		tfRefVal, valueTemplate, containsTFRef, err := sampleconversion.GetTFReferenceValue(value)
   436  		if err != nil {
   437  			return nil, nil, fmt.Errorf("error getting TF reference value for field %v: %w", tfFieldName, err)
   438  		}
   439  		if containsTFRef {
   440  			dependencyGraph.AddDependencyWithTFRefVal(tfRefVal, tfType)
   441  			// The use case that the field has a reference in the Tf sample, but
   442  			// is not a reference field in the KRM resource can't be handled.
   443  			refConfig, ok := refConfigMap[tfFieldName]
   444  			if !ok {
   445  				krmRefVal, err := sampleconversion.ConstructKRMExternalRefValFromTFRefVal(tfRefVal, valueTemplate, tfToGVK)
   446  				if err != nil {
   447  					return nil, nil, fmt.Errorf("cannot construct KRM value for a TF reference field %v: %w", krmFieldName, err)
   448  				}
   449  				krmSpec[krmFieldName] = krmRefVal
   450  				continue
   451  			}
   452  
   453  			krmFieldName = refConfig.Key
   454  			// For organizational resources that shouldn't be created, but used
   455  			// as a reference, use the default one instead.
   456  			defaultVal, ok := defaultOrganizationalResourcesMap[refConfig.GVK.Kind]
   457  			if ok {
   458  				krmRefVal := sampleconversion.ConstructKRMExternalReferenceObject(defaultVal)
   459  				krmSpec[krmFieldName] = krmRefVal
   460  				continue
   461  			}
   462  
   463  			krmRefVal, err := sampleconversion.ConstructKRMNameReferenceObject(tfRefVal, tfToGVK)
   464  			if err != nil {
   465  				return nil, nil, fmt.Errorf("error constructing KRM reference value for field %v: %w", krmFieldName, err)
   466  			}
   467  			krmSpec[krmFieldName] = krmRefVal
   468  			continue
   469  		}
   470  		if isProjectNameWithNumber(value) {
   471  			testProjectNameWithNumber := "projects/${projectNumber}"
   472  			testProjectID := "${projectId}"
   473  			// It's possible that the field with a value of the relative
   474  			// resource name of a GCP project is a reference field in KRM.
   475  			refConfig, ok := refConfigMap[tfFieldName]
   476  			if !ok {
   477  				container, ok := containerMap[tfFieldName]
   478  				if !ok {
   479  					krmSpec[krmFieldName] = testProjectNameWithNumber
   480  					continue
   481  				}
   482  				if !isProjectContainer(container) {
   483  					return nil, nil, fmt.Errorf("expected container type for field %v to be project but is %+v", tfFieldName, container.Type)
   484  				}
   485  				if len(containerAnnotation) > 0 {
   486  					return nil, nil, fmt.Errorf("more than one container annotation found: '%+v' and '%v: %v'", containerAnnotation, tfFieldName, testProjectNameWithNumber)
   487  				}
   488  				containerAnnotation[k8s.GetAnnotationForContainerType(container.Type)] = testProjectID
   489  				continue
   490  			}
   491  			krmFieldName = refConfig.Key
   492  			krmRefVal := sampleconversion.ConstructKRMExternalReferenceObject(testProjectNameWithNumber)
   493  			krmSpec[krmFieldName] = krmRefVal
   494  			continue
   495  		}
   496  		if isOrganizationName(value) {
   497  			testOrgID := "${TEST_ORG_ID}"
   498  			testOrgName := fmt.Sprintf("organizations/%v", testOrgID)
   499  			// It's possible that the field with a value of the relative
   500  			// resource name of a GCP organization is a reference field in KRM.
   501  			refConfig, ok := refConfigMap[tfFieldName]
   502  			if !ok {
   503  				container, ok := containerMap[tfFieldName]
   504  				if !ok {
   505  					krmSpec[krmFieldName] = testOrgName
   506  					continue
   507  				}
   508  				if !isOrganizationContainer(container) {
   509  					return nil, nil, fmt.Errorf("expected container type for field %v to be organization but is %+v", tfFieldName, container.Type)
   510  				}
   511  				if len(containerAnnotation) > 0 {
   512  					return nil, nil, fmt.Errorf("more than one container annotation found: '%+v' and '%v: %v'", containerAnnotation, tfFieldName, testOrgName)
   513  				}
   514  				containerAnnotation[k8s.GetAnnotationForContainerType(container.Type)] = testOrgID
   515  				continue
   516  			}
   517  			krmFieldName = refConfig.Key
   518  			krmRefVal := sampleconversion.ConstructKRMExternalReferenceObject(testOrgName)
   519  			krmSpec[krmFieldName] = krmRefVal
   520  			continue
   521  		}
   522  		result, err := krmifyNestedField(value)
   523  		if err != nil {
   524  			return nil, nil, fmt.Errorf("error krmifying the nested field %s: %w", krmFieldName, err)
   525  		}
   526  		krmSpec[krmFieldName] = result
   527  	}
   528  	return krmSpec, containerAnnotation, nil
   529  }
   530  
   531  func krmifyNestedField(value interface{}) (interface{}, error) {
   532  	switch value.(type) {
   533  	case []interface{}:
   534  		arrayValue := value.([]interface{})
   535  		krmArray := make([]interface{}, 0)
   536  		for _, v := range arrayValue {
   537  			result, err := krmifyNestedField(v)
   538  			if err != nil {
   539  				return nil, fmt.Errorf("error krmifying the array field: %w", err)
   540  			}
   541  			krmArray = append(krmArray, result)
   542  		}
   543  		return krmArray, nil
   544  	case map[string]interface{}:
   545  		mapValue := value.(map[string]interface{})
   546  		krmMap := make(map[string]interface{})
   547  		for k, v := range mapValue {
   548  			krmFieldName := text.SnakeCaseToLowerCamelCase(k)
   549  			result, err := krmifyNestedField(v)
   550  			if err != nil {
   551  				return nil, fmt.Errorf("error krmifying the object field: %w", err)
   552  			}
   553  			krmMap[krmFieldName] = result
   554  		}
   555  		return krmMap, nil
   556  	default:
   557  		// TODO(b/265367198): Handle nested reference fields.
   558  		return value, nil
   559  	}
   560  }
   561  
   562  func handleKRMFields(spec map[string]interface{}, containerAnnotation map[string]string, gvk schema.GroupVersionKind, rc v1alpha1.ResourceConfig) (map[string]interface{}, error) {
   563  	krmStruct := make(map[string]interface{})
   564  	krmStruct["apiVersion"] = gvk.GroupVersion().String()
   565  	krmStruct["kind"] = gvk.Kind
   566  
   567  	metadata := make(map[string]interface{})
   568  	metadata["name"] = fmt.Sprintf("%s-${uniqueId}", strings.ToLower(gvk.Kind))
   569  
   570  	// Fields mapping to `metadata.name` and `metadata.labels` should be removed
   571  	// from `spec`.
   572  	nameField := text.SnakeCaseToLowerCamelCase(rc.MetadataMapping.Name)
   573  	labelsField := text.SnakeCaseToLowerCamelCase(rc.MetadataMapping.Labels)
   574  	if _, ok := spec[nameField]; ok {
   575  		delete(spec, nameField)
   576  	}
   577  	labels, ok := spec[labelsField]
   578  	if ok {
   579  		metadata["labels"] = labels
   580  		delete(spec, labelsField)
   581  	}
   582  
   583  	if len(containerAnnotation) > 1 {
   584  		return nil, fmt.Errorf("more than one container annotation provided: %+v", containerAnnotation)
   585  	}
   586  	for key, value := range containerAnnotation {
   587  		annotations := make(map[string]interface{})
   588  		annotations[key] = value
   589  		metadata["annotations"] = annotations
   590  	}
   591  	krmStruct["metadata"] = metadata
   592  
   593  	// Setting alphanumeric resourceID for all resources that support the
   594  	// user-specified resourceID field to avoid the edge cases when a resource
   595  	// has a different naming convention from K8s objects'.
   596  	if rc.ResourceID.TargetField != "" && rc.ServerGeneratedIDField == "" {
   597  		resourceID, err := generateResourceID(gvk.Kind)
   598  		if err != nil {
   599  			return nil, fmt.Errorf("error generating resource ID for kind %v: %w", gvk.Kind, err)
   600  		}
   601  		spec["resourceID"] = resourceID
   602  	}
   603  
   604  	// Hierarchical references should be represented as reference fields instead.
   605  	var hierarchicalReferenceConfigured bool
   606  	supportedHierarchicalReferenceTypes := make(map[v1alpha1.HierarchicalReferenceType]bool)
   607  	for _, hr := range rc.HierarchicalReferences {
   608  		refConfig, err := krmtotf.GetReferenceConfigForHierarchicalReference(hr, &rc)
   609  		if err != nil {
   610  			return nil, fmt.Errorf("error retrieving reference config: %w", err)
   611  		}
   612  		supportedHierarchicalReferenceTypes[hr.Type] = true
   613  		tfField := text.SnakeCaseToLowerCamelCase(refConfig.TFField)
   614  		_, ok := spec[tfField]
   615  		if !ok {
   616  			continue
   617  		}
   618  		switch hr.Type {
   619  		case v1alpha1.HierarchicalReferenceTypeProject:
   620  			refVal := make(map[string]interface{})
   621  			refVal["name"] = "project-${uniqueId}"
   622  			spec["projectRef"] = refVal
   623  		case v1alpha1.HierarchicalReferenceTypeFolder:
   624  			spec["folderRef"] = map[string]string{"external": "${TEST_FOLDER_ID}"}
   625  		case v1alpha1.HierarchicalReferenceTypeOrganization:
   626  			spec["organizationRef"] = map[string]string{"external": "${TEST_ORG_ID}"}
   627  		default:
   628  			return nil, fmt.Errorf("unsupported hierarchical reference type: %v", hr.Type)
   629  		}
   630  		delete(spec, tfField)
   631  		hierarchicalReferenceConfigured = true
   632  	}
   633  	// If a resource has hierarchical reference field(s), but the field is not
   634  	// explicitly configured, it means that the TF sample uses the default
   635  	// project configured by the TF provider.
   636  	// We need to add the `projectRef` field explicitly.
   637  	if _, ok := supportedHierarchicalReferenceTypes[v1alpha1.HierarchicalReferenceTypeProject]; ok && !hierarchicalReferenceConfigured {
   638  		spec["projectRef"] = map[string]string{"external": "${projectId}"}
   639  	}
   640  
   641  	// TODO(b/265367279): Handle nested special field.
   642  	spec = handleSpecialTopLevelFields(spec, gvk.Kind)
   643  	krmStruct["spec"] = spec
   644  
   645  	return krmStruct, nil
   646  }
   647  
   648  func handleSpecialTopLevelFields(spec map[string]interface{}, kind string) map[string]interface{} {
   649  	nonIDFieldsRequiringUniqueValues, _ := nonIDKRMFieldsRequiringUniqueValuesMap[kind]
   650  	fieldsNotAllowingSpecialChars, _ := krmFieldsNotAllowingSpecialCharsMap[kind]
   651  	listFieldsWithAtMostOneItemMap, _ := ListFieldsWithAtMostOneItemMap[kind]
   652  	if len(nonIDFieldsRequiringUniqueValues) == 0 &&
   653  		len(fieldsNotAllowingSpecialChars) == 0 &&
   654  		len(listFieldsWithAtMostOneItemMap) == 0 {
   655  		return spec
   656  	}
   657  
   658  	updatedSpec := make(map[string]interface{})
   659  	for fieldName, value := range spec {
   660  		switch value.(type) {
   661  		case string:
   662  			strVal := value.(string)
   663  
   664  			if len(nonIDFieldsRequiringUniqueValues) > 0 {
   665  				if _, ok := nonIDFieldsRequiringUniqueValues[fieldName]; ok {
   666  					strVal += uniqueIDHolder
   667  				}
   668  			}
   669  
   670  			if len(fieldsNotAllowingSpecialChars) > 0 {
   671  				if _, ok := fieldsNotAllowingSpecialChars[fieldName]; ok {
   672  					strVal = text.RemoveSpecialCharacters(strVal)
   673  				}
   674  			}
   675  			updatedSpec[fieldName] = strVal
   676  		case []interface{}:
   677  			listVal := value.([]interface{})
   678  			if _, ok := listFieldsWithAtMostOneItemMap[fieldName]; ok {
   679  				updatedSpec[fieldName] = listVal[0]
   680  			}
   681  		default:
   682  			updatedSpec[fieldName] = value
   683  		}
   684  	}
   685  	return updatedSpec
   686  }
   687  
   688  func GetTFTypeToGVKMap(smLoader *servicemappingloader.ServiceMappingLoader) (map[string]schema.GroupVersionKind, error) {
   689  	tfTypeToGVK := make(map[string]schema.GroupVersionKind)
   690  	for _, sm := range smLoader.GetServiceMappings() {
   691  		for _, rc := range sm.Spec.Resources {
   692  			tfType := rc.Name
   693  			gvk := schema.GroupVersionKind{
   694  				Group:   sm.Name,
   695  				Version: sm.GetVersionFor(&rc),
   696  				Kind:    rc.Kind,
   697  			}
   698  			tfTypeToGVK[tfType] = gvk
   699  		}
   700  	}
   701  	return tfTypeToGVK, nil
   702  }
   703  
   704  func insertTestData(createConfig map[string]interface{}, dependenciesConfig []map[string]interface{}, autoGenType *allowlist.AutoGenType, sampleName string, generatedSamples map[string]bool) error {
   705  	folderPath := getTestDataFolderPath(autoGenType)
   706  
   707  	createFilePath := filepath.Join(folderPath, sampleName, "create.yaml")
   708  	if err := os.MkdirAll(filepath.Dir(createFilePath), 0770); err != nil {
   709  		return fmt.Errorf("error creating folder for path %v: %v", createFilePath, err)
   710  	}
   711  	createConfigInBytes, err := yaml.Marshal(createConfig)
   712  	if err != nil {
   713  		return fmt.Errorf("err marshaling createConfig to yaml: %w", err)
   714  	}
   715  	if err := os.WriteFile(createFilePath, createConfigInBytes, 0644); err != nil {
   716  		return fmt.Errorf("error writing to file %v: %w", createFilePath, err)
   717  	}
   718  
   719  	if len(dependenciesConfig) > 0 {
   720  		dependenciesFilePath := filepath.Join(folderPath, sampleName, "dependencies.yaml")
   721  		var dependenciesConfigInBytes []byte
   722  
   723  		for i, r := range dependenciesConfig {
   724  			resourceConfigInBytes, err := yaml.Marshal(r)
   725  			if err != nil {
   726  				return fmt.Errorf("err marshaling resource config in dependencies to yaml: %w", err)
   727  			}
   728  			if i != 0 {
   729  				yamlSeparator := []byte("---\n")
   730  				dependenciesConfigInBytes = append(dependenciesConfigInBytes, yamlSeparator...)
   731  			}
   732  			dependenciesConfigInBytes = append(dependenciesConfigInBytes, resourceConfigInBytes...)
   733  		}
   734  		if err := os.WriteFile(dependenciesFilePath, dependenciesConfigInBytes, 0644); err != nil {
   735  			return fmt.Errorf("error writing to file %v: %w", dependenciesFilePath, err)
   736  		}
   737  	}
   738  	return nil
   739  }
   740  
   741  func getTestDataFolderPath(autoGenType *allowlist.AutoGenType) string {
   742  	serviceFolderName := autoGenType.ServiceNameInLC
   743  	kindFolderName := strings.ToLower(autoGenType.KRMKindName)
   744  	return filepath.Join(repo.GetBasicIntegrationTestDataPath(), serviceFolderName, autoGenType.Version, kindFolderName)
   745  }
   746  
   747  func getReferenceConfigMap(rc v1alpha1.ResourceConfig) map[string]v1alpha1.ReferenceConfig {
   748  	refConfigMap := make(map[string]v1alpha1.ReferenceConfig)
   749  	for _, refConfig := range rc.ResourceReferences {
   750  		refConfigMap[refConfig.TFField] = refConfig
   751  	}
   752  	return refConfigMap
   753  }
   754  
   755  func getContainerMap(rc v1alpha1.ResourceConfig) map[string]v1alpha1.Container {
   756  	containerMap := make(map[string]v1alpha1.Container)
   757  	for _, container := range rc.Containers {
   758  		containerMap[container.TFField] = container
   759  	}
   760  	return containerMap
   761  }
   762  
   763  func isOrganizationName(value interface{}) bool {
   764  	str, ok := value.(string)
   765  	if !ok {
   766  		return false
   767  	}
   768  	orgNameRegex := regexp.MustCompile(`^organizations/[0-9]{5,15}$`)
   769  	matchResult := orgNameRegex.FindStringSubmatch(str)
   770  	if len(matchResult) == 1 {
   771  		return true
   772  	}
   773  	return false
   774  }
   775  
   776  func isProjectNameWithNumber(value interface{}) bool {
   777  	str, ok := value.(string)
   778  	if !ok {
   779  		return false
   780  	}
   781  	projectNameRegex := regexp.MustCompile(`^projects/[0-9]{5,15}$`)
   782  	matchResult := projectNameRegex.FindStringSubmatch(str)
   783  	if len(matchResult) == 1 {
   784  		return true
   785  	}
   786  	return false
   787  }
   788  
   789  func isProjectContainer(container v1alpha1.Container) bool {
   790  	switch container.Type {
   791  	case v1alpha1.ContainerTypeProject:
   792  		return true
   793  	default:
   794  		return false
   795  	}
   796  }
   797  
   798  func isOrganizationContainer(container v1alpha1.Container) bool {
   799  	switch container.Type {
   800  	case v1alpha1.ContainerTypeOrganization:
   801  		return true
   802  	default:
   803  		return false
   804  	}
   805  }
   806  
   807  func generateResourceID(kind string) (string, error) {
   808  	supportedLength, ok := ResourceIDLengthMap[kind]
   809  	resourceID := fmt.Sprintf("%s${uniqueId}", strings.ToLower(kind))
   810  	if !ok {
   811  		return resourceID, nil
   812  	}
   813  
   814  	// The generated unique ID has 20 characters.
   815  	// If the supported length of the ID for the resource is shorter than 20
   816  	// characters, we can't convert the resource sample.
   817  	// Otherwise, remove the first X letters of the resourceID to fit.
   818  	if supportedLength <= 20 {
   819  		return "", fmt.Errorf("supported resource ID supportedLength should > 20")
   820  	}
   821  	resourceIDLength := len(kind) + 20
   822  	var numberOfLettersToRemove int
   823  	if resourceIDLength > supportedLength {
   824  		numberOfLettersToRemove = resourceIDLength - supportedLength
   825  	}
   826  	return resourceID[numberOfLettersToRemove:], nil
   827  }
   828
View as plain text