1 // Copyright 2022 Google LLC 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 //go:build integration 16 // +build integration 17 18 package create 19 20 import ( 21 "context" 22 "flag" 23 "regexp" 24 "testing" 25 26 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/kccmanager" 27 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/controller/registration" 28 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/logging" 29 testgcp "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/gcp" 30 testmain "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/test/main" 31 32 "golang.org/x/sync/semaphore" 33 klog "sigs.k8s.io/controller-runtime/pkg/log" 34 "sigs.k8s.io/controller-runtime/pkg/manager" 35 "sigs.k8s.io/controller-runtime/pkg/manager/signals" 36 ) 37 38 func init() { 39 // run-tests allows you to limit the tests that are run by specifying 40 // regexes to be used to match test names. The "test name" in this case 41 // corresponds to the directory name of the sample YAMLs. 42 flag.StringVar(&runTestsRegex, "run-tests", "", "run only the tests whose names match the given regex") 43 // cleanup-resources allows you to disable the cleanup of resources created during testing. This can be useful for debugging test failures. 44 // The default value is true. 45 // 46 // To use this flag, you MUST use an equals sign as follows: go test -tags=integration -cleanup-resources=false 47 flag.BoolVar(&cleanupResources, "cleanup-resources", true, "when enabled, "+ 48 "cloud resources created by tests will be cleaned up at the end of a test") 49 } 50 51 var ( 52 runTestsRegex string 53 cleanupResources bool 54 mgr manager.Manager 55 // this manager is only used to get the rest.Config from the test framework 56 unusedManager manager.Manager 57 58 logger = klog.Log 59 ) 60 61 var testDisabledList = map[string]bool{ 62 // Long-term denylist -- current test framework does not support these tests 63 // 64 // The resources below have special test requirements 65 "computeinterconnectattachment": true, 66 "firestoreindex": true, 67 // The below test requires more resources than the default quota allows 68 "computenodegroup": true, 69 "sole-tenant-node-pool": true, 70 // Unsupported IAMPolicy tests 71 "workload-identity-policy": true, // needs a real cluster 72 // Without the ignore-warnings directive, trying to delete the app profile without another one configured will fail. 73 // However, ignore-warnings set to true for a single cluster profile is probably not a good idea. 74 "single-cluster-bigtable-app-profile": true, 75 // The below tests require logging into the superadmin account on the KCC test 76 // organization, and assigning a domain admin role to the service 77 // account used by the samples test runner. 78 // This authentication can't be automated in our test set-up script. 79 "cloudidentitygroup": true, 80 "membership-with-expiration-date": true, 81 "membership-with-manager-role": true, 82 // The following tests require using a Service Account under an allowlisted project (with 83 // label WORKFORCE_POOLS_TRUSTED_TESTER). 84 "iamworkforcepool": true, 85 "oidc-workforce-pool-provider": true, 86 "saml-workforce-pool-provider": true, 87 // The following tests require using a Service Account under an allowlisted project (with 88 // label ACCESS_BOUNDARY_TRUSTED_TESTER). 89 "iamaccessboundarypolicy": true, 90 // The samples below requires a custom project with the test billing account 91 // configured, which has a quota limit on the number of projects attached to 92 // it. We should disable redundant sample tests for the same resource. 93 // 94 // Disable 4 out of 5 BinaryAuthorizationPolicy samples. 95 "default-policy": true, 96 "namespace-policy": true, 97 "service-account-policy": true, 98 "service-identity-policy": true, 99 100 // The below tests should stay denylisted until underlying issues are fixed 101 // 102 // Access Context Manager access policies are a singleton for an entire organization; 103 // we need to keep on persistently around to test the access level and service policy 104 // resources, so we shouldn't modify our existing one. 105 "accesscontextmanageraccesspolicy": true, 106 "accesscontextmanageraccesslevel": true, 107 // Behaviour of Service Perimeter is similar to access level and service policy. 108 // So disabling testing for Service Perimeter 109 "accesscontextmanagerserviceperimeter": true, 110 // Cloud Build Triggers for GitHub repos require the user to connect their 111 // GCP project to their GitHub repo first to work. 112 "build-trigger-for-github-repo": true, 113 // Samples test appends a 4-character UID to the end of project name, making it too long 114 "computesharedvpcserviceproject": true, 115 // DNS sample testing will cause samples to contend with previous attempts to create records for their domain name. 116 "dns-a-record-set": true, 117 "dns-aaaa-record-set": true, 118 "dns-cname-record-set": true, 119 "dns-mx-record-set": true, 120 "dns-ns-record-set": true, 121 "dns-srv-record-set": true, 122 "dns-txt-record-set": true, 123 "dnssec-dnskey-record-set": true, 124 "dnssec-ds-record-set": true, 125 "dnssec-ipsecvpnkey-record-set": true, 126 "dnssec-sshfp-record-set": true, 127 // The GSA that authenticates KCC has to be the owner of a Google Group in order to use the group email address as 128 // the valid support email. In the integration test, we use the email address of the GSA to bypass the support email validation by IAP REST API. 129 // In samples, we want to show the real use case with group/user email address. 130 "iapbrand": true, 131 "iapidentityawareproxyclient": true, 132 // Utilizes IAPBrand as a dependency 133 "oauth2clientid-backend-service": true, 134 // SQL instance replication has complex dependencies on network connection which are unsupported 135 "mysql-sql-instance-with-replication": true, 136 // Org-level audit configs are uniquely identified by the org + service 137 // combination, which means there will be collisions if we allow multiple 138 // test runs executing in parallel to create IAMAuditConfig resources for 139 // the same org and service combination. 140 "external-organization-level-audit-config": true, 141 // The sample below creates an org-level role. Since we currently don't 142 // have the appropriate logic to periodically clean up leaked org level 143 // roles, block this sample for now to mitigate maximizing the org's 144 // IAMCustomRole quota (b/177688780). 145 "organization-role": true, 146 // The sample below creates an IAMPolicyMember that uses an org-level role. 147 // Org-level roles cannot be recreated for 7-37 days after being deleted, 148 // so sample test runs that inadvertently re-use the same role ID may fail 149 // since you cannot create IAMPolicyMembers that use deleted roles. 150 "org-level-iam-custom-role-policy-member": true, 151 // Org-level org policy is uniquely identified by the org + constraint combination, 152 // which affects overall org behavior and can't be safely tested from multiple 153 // processes running simultaneously. 154 "organization-policy-for-organization": true, 155 // The below tests require manually enabling the Google Identity Platform in the Google Cloud Console. 156 "identityplatformoauthidpconfig": true, 157 "identityplatformconfig": true, 158 "identityplatformtenant": true, 159 "identityplatformtenantoauthidpconfig": true, 160 // TODO(b/193158731): Remove the sample below after understanding why Network can't be cleaned up. 161 "vpc-native-container-cluster": true, 162 // Deleting the ContainerCluster can fail if its Project was deleted first. 163 // Disable these samples for now until ContainerCluster supports 164 // hierarchical references or once we support proper deletion ordering 165 // (b/179907721). 166 "config-management-feature-membership": true, 167 "service-mesh-feature-membership": true, 168 "multi-cluster-ingress-feature": true, 169 // There is a quota limit of only 10 FirewallPolicies globally in an org, 170 // these samples tests are omitted to minimize the number of FirewallPolicies created. 171 "computefirewallpolicy": true, 172 "computefirewallpolicyrule": true, 173 "association-with-folder-attachment-target": true, 174 // In addition to wanting to limit the number of FirewallPolicies we create for 175 // the above reason, we want to disable association-with-organization-attachment-target 176 // because each organization can only be associated with one ComputeFirewallPolicyAssociation 177 // at a given time, and we can't create and delete new organizations for test 178 // purposes. Therefore, in order to avoid conflicts when we have multiple 179 // presubmit tests running in parallel, this test needs to be disabled. 180 "association-with-organization-attachment-target": true, 181 // The below tests require the project to be added to the allowlist for resource creation. 182 "android-recaptcha-enterprise-key": true, 183 "ios-recaptcha-enterprise-key": true, 184 // This sample uses cloud source repo, as a result it is required: 185 // 1. The deployer account needs to have access to the repo. 186 // 2. The repo contains to-be-deployed source code. 187 // This setup cannot be easily done in sample tests. Also due to issue 1, customers cannot directly 188 // apply this sample before updating it with a proper cloud source repo URL. 189 "eventtrigger-with-pubsubtopic": true, 190 // When testing these samples, they always error out on deletion due to a lack 191 // of ordered deletion of dependencies. I.e., privatecacapool is deleted before 192 // privatecacertificateauthority (which has privatecacapool as a dependency), 193 // so privatecacertificateauthority fails on deletion (b/225964552). 194 "basic-certificate": true, 195 "cert-sign-certificate": true, 196 "complex-certificate": true, 197 // TODO(b/240327420): The ApigeeEnvironment test seems to be timing out, 198 // thereby causing the samples test to fail and blocking the release. 199 // Disable the test for now and re-enable it once the issue has been 200 // resolved. 201 "apigeeenvironment": true, 202 // This sample test is failing because of parallel deletion failure from the API. 203 // Disable the test for now while we are figuring out the long term fix with the 204 // service team (b/260214463). 205 "private-service-connection-region-network-endpoint-group": true, 206 // This sample test is failing because configconnector.net GCP org is not allowlisted. 207 // Disable the test until we have fixed b/267510222. 208 "calendar-budget": true, 209 // Temporary list to disable because of billing policy change. To be recovered in b/276980291 210 "anthos-config-management-feature": true, 211 "anthos-service-mesh-feature": true, 212 "billing-account-log-bucket": true, 213 "billing-exclusion": true, 214 "cluster-policy": true, 215 "external-project-level-policy": true, 216 "multi-cluster-service-discovery-feature": true, 217 "organization-policy-for-project": true, 218 "project-exclusion": true, 219 "project-in-folder": true, 220 "project-in-org": true, 221 "project-level-policy": true, 222 "project-sink": true, 223 } 224 225 func TestAll(t *testing.T) { 226 project := testgcp.GetDefaultProject(t) 227 228 setup() 229 samples := loadSamplesOntoUnstructs(t, regexp.MustCompile(runTestsRegex), project) 230 // Sort the samples in descending order by number of resources. This is an attempt to start the samples that use 231 // a network and have many dependencies sooner since they will likely be the longest running. 232 sortSamplesInDescendingOrderByNumberOfResources(samples) 233 // limit the number of in-progress test that have a network to 14. 15 networks is the maximum number of networks 234 // in a newly created GCP project for our billing account's "reputation". There is one default network leaving room 235 // for 14 more networks. The reason to limit the number of tests is to avoid lengthening the test runtime due to 236 // extended exponential backoff from failure to create a network due to quota. 237 maximumNetworks := 14 238 sem := semaphore.NewWeighted(int64(maximumNetworks)) 239 releaseFunc := func(s Sample, n int64) { 240 logger.Info("Releasing network semaphore for test", "testName", s.Name) 241 sem.Release(n) 242 } 243 for _, s := range samples { 244 if _, ok := testDisabledList[s.Name]; ok { 245 continue 246 } 247 s := s 248 s.Resources = replaceResourceNamesWithUniqueIDs(t, s.Resources) 249 t.Run(s.Name, func(t *testing.T) { 250 t.Parallel() 251 252 ctx := context.TODO() 253 254 h := NewHarnessWithManager(t, ctx, mgr) 255 SetupNamespacesAndApplyDefaults(h, []Sample{s}, project) 256 257 networkCount := int64(networksInSampleCount(s)) 258 if networkCount > 0 { 259 logger.Info("Acquiring network semaphore for test...", "testName", s.Name) 260 if err := sem.Acquire(context.TODO(), networkCount); err != nil { 261 t.Fatalf("error acquiring semaphore: %v", err) 262 } 263 logger.Info("Acquired network semaphore for test", "testName", s.Name) 264 defer releaseFunc(s, networkCount) 265 } 266 RunCreateDeleteTest(h, s.Resources, cleanupResources) 267 }) 268 } 269 } 270 271 func setup() { 272 ctx := context.TODO() 273 flag.Parse() 274 var err error 275 mgr, err = kccmanager.New(ctx, unusedManager.GetConfig(), kccmanager.Config{}) 276 if err != nil { 277 logging.Fatal(err, "error creating new manager") 278 } 279 // Register the deletion defender controller 280 if err := registration.Add(mgr, nil, nil, nil, nil, registration.RegisterDeletionDefenderController); err != nil { 281 logging.Fatal(err, "error adding registration controller for deletion defender controllers") 282 } 283 // start the manager, Start(...) is a blocking operation so it needs to be done asynchronously 284 go func() { 285 if err := mgr.Start(signals.SetupSignalHandler()); err != nil { 286 logging.Fatal(err, "error starting manager") 287 } 288 }() 289 } 290 291 func TestMain(m *testing.M) { 292 testmain.TestMainForIntegrationTests(m, &unusedManager) 293 } 294