1 package clusterctl
2
3 import (
4 "context"
5 "errors"
6 "fmt"
7 "strings"
8 "sync"
9 "time"
10
11 containerAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/container/v1beta1"
12 "github.com/go-logr/logr"
13 corev1 "k8s.io/api/core/v1"
14 kerrors "k8s.io/apimachinery/pkg/api/errors"
15 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
16 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
17 k8sRuntime "k8s.io/apimachinery/pkg/runtime"
18 kuberecorder "k8s.io/client-go/tools/record"
19 ctrl "sigs.k8s.io/controller-runtime"
20 "sigs.k8s.io/controller-runtime/pkg/client"
21 "sigs.k8s.io/controller-runtime/pkg/controller"
22 "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
23 "sigs.k8s.io/controller-runtime/pkg/event"
24 "sigs.k8s.io/controller-runtime/pkg/predicate"
25
26 "edge-infra.dev/pkg/edge/api/totp"
27 gkeClusterApi "edge-infra.dev/pkg/edge/apis/gkecluster/v1alpha1"
28 bs "edge-infra.dev/pkg/edge/bootstrapping"
29 "edge-infra.dev/pkg/edge/constants"
30 clusterConstants "edge-infra.dev/pkg/edge/constants/api/cluster"
31 "edge-infra.dev/pkg/edge/constants/api/fleet"
32 "edge-infra.dev/pkg/edge/k8objectsutils"
33 "edge-infra.dev/pkg/edge/registration"
34 ipranger "edge-infra.dev/pkg/f8n/ipranger/server"
35 "edge-infra.dev/pkg/k8s/konfigkonnector/apis/meta"
36 "edge-infra.dev/pkg/k8s/meta/status"
37 "edge-infra.dev/pkg/k8s/runtime/conditions"
38 "edge-infra.dev/pkg/k8s/runtime/controller/metrics"
39 "edge-infra.dev/pkg/k8s/runtime/controller/reconcile"
40 "edge-infra.dev/pkg/k8s/runtime/controller/reconcile/recerr"
41 "edge-infra.dev/pkg/k8s/runtime/inventory"
42 "edge-infra.dev/pkg/k8s/runtime/patch"
43 "edge-infra.dev/pkg/k8s/runtime/sap"
44 unstructuredutil "edge-infra.dev/pkg/k8s/unstructured"
45 ff "edge-infra.dev/pkg/lib/featureflag"
46 "edge-infra.dev/pkg/lib/gcp/iam"
47 "edge-infra.dev/pkg/lib/uuid"
48 )
49
50 const (
51 ErrCreatingContainerCluster = "failed to create ContainerCluster"
52 ErrGettingContainerCluster = "failed to get ContainerCluster"
53 ErrCreatingContainerNodePool = "failed to create ContainerNodePool"
54 ErrGettingContainerNodePool = "failed to get ContainerNodePool"
55 ErrGKEClusterStatusUpdate = "unable to update GKECluster status"
56 ErrContainerClusterNotReady = "ContainerCluster not yet ready"
57 ErrContainerNodePoolNotReady = "ContainerNodePool not yet ready"
58 ErrEdgeBootstrapAPIFailed = "edge-bootstrap api failed"
59 ErrInvalidGKECluster = "invalid GKECluster spec"
60 ErrCreatingConfigConnector = "failed to create config connector resource"
61 )
62
63 var (
64 GKEClusterLabel = fmt.Sprintf("%s.%s", strings.ToLower(gkeClusterApi.Kind), gkeClusterApi.GKEClusterGVK.Group)
65 )
66
67 type EdgeClusterConfig struct {
68 Location string
69 MachineType string
70 NumNode int
71 MinNodes int
72 MaxNodes int
73 Autoscale bool
74 }
75
76 var gkeClientCache = struct {
77 sync.Mutex
78 c map[string]client.Client
79 }{
80 c: make(map[string]client.Client),
81 }
82
83 var clusterConfigs = map[string]EdgeClusterConfig{
84 fleet.Cluster: {
85 Location: "us-east1-c",
86 MachineType: "e2-highmem-4",
87 NumNode: 2,
88 MinNodes: 1,
89 MaxNodes: 3,
90 Autoscale: true,
91 },
92 fleet.Store: {
93 Location: "us-east1-c",
94 MachineType: "e2-highmem-4",
95 NumNode: 2,
96 MinNodes: 1,
97 MaxNodes: 3,
98 Autoscale: true,
99 },
100 fleet.CouchDB: {
101 Location: "us-east1-c",
102 MachineType: "n2d-highcpu-8",
103 NumNode: 2,
104 MinNodes: 1,
105 MaxNodes: 3,
106 Autoscale: true,
107 },
108 fleet.BasicStore: {
109 Location: "us-east1-c",
110 MachineType: "e2-standard-4",
111 NumNode: 2,
112 MinNodes: 1,
113 MaxNodes: 3,
114 Autoscale: true,
115 },
116 }
117
118
119
120
121 var gkeClusterConditions = reconcile.Conditions{
122 Target: status.ReadyCondition,
123 Owned: []string{
124 status.ReadyCondition,
125 status.ReconcilingCondition,
126 status.StalledCondition,
127 },
128 Summarize: []string{
129 status.StalledCondition,
130 },
131 NegativePolarity: []string{
132 status.ReconcilingCondition,
133 status.StalledCondition,
134 },
135 }
136
137 type ContainerClusterClientFunc func(cluster containerAPI.ContainerCluster, o client.Options) (client.Client, error)
138
139
140 type GKEClusterReconciler struct {
141 client.Client
142 kuberecorder.EventRecorder
143 manager ctrl.Manager
144 Log logr.Logger
145 Metrics metrics.Metrics
146 Scheme *k8sRuntime.Scheme
147 CreateClient ContainerClusterClientFunc
148 EdgeAPI string
149 IPRangerClient *ipranger.Client
150 DefaultRequeue time.Duration
151 WaitForSetTimeout time.Duration
152 TopLevelProject string
153 TopLevelCNRMSA string
154 TotpSecret string
155 ResourceManager *sap.ResourceManager
156 Name string
157 Conditions reconcile.Conditions
158 Concurrency int
159 }
160
161 func gkeClusterPredicate() predicate.Predicate {
162 return predicate.Funcs{
163 UpdateFunc: func(_ event.UpdateEvent) bool {
164 return false
165 },
166 CreateFunc: func(_ event.CreateEvent) bool {
167 return true
168 },
169 DeleteFunc: func(_ event.DeleteEvent) bool {
170 return false
171 },
172 }
173 }
174
175
176 func (r *GKEClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
177 return ctrl.NewControllerManagedBy(mgr).
178 For(&gkeClusterApi.GKECluster{}).
179 WithOptions(controller.Options{
180 MaxConcurrentReconciles: r.Concurrency,
181 }).
182 WithEventFilter(gkeClusterPredicate()).
183 Complete(r)
184 }
185
186 func (r *GKEClusterReconciler) PatchOpts() []patch.Option {
187 return []patch.Option{
188 patch.WithOwnedConditions{Conditions: r.Conditions.Owned},
189 patch.WithFieldOwner(r.Name),
190 }
191 }
192
193
194
195 func (r *GKEClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, recErr error) {
196 var (
197 reconcileStart = time.Now()
198 log = ctrl.LoggerFrom(ctx)
199 result = reconcile.ResultEmpty
200 gkeCluster = &gkeClusterApi.GKECluster{}
201 )
202
203 if err := r.Client.Get(ctx, req.NamespacedName, gkeCluster); err != nil {
204 return ctrl.Result{}, client.IgnoreNotFound(err)
205 }
206 r.Metrics.RecordReconciling(ctx, gkeCluster)
207 updateReconcileMetadata(ctx, gkeCluster, reconcileStart)
208
209 oldStatus := gkeCluster.Status.DeepCopy()
210 patcher := patch.NewSerialPatcher(gkeCluster, r.Client)
211
212 defer func() {
213 if recErrToCondition, ok := recErr.(recerr.Error); ok {
214 recErrToCondition.ToCondition(gkeCluster, status.ReadyCondition)
215 }
216
217 summarizer := reconcile.NewSummarizer(patcher)
218 res, recErr = summarizer.SummarizeAndPatch(ctx, gkeCluster, []reconcile.SummarizeOption{
219 reconcile.WithConditions(r.Conditions),
220 reconcile.WithError(recErr),
221 reconcile.WithResult(result),
222 reconcile.WithIgnoreNotFound(),
223 reconcile.WithProcessors(
224 reconcile.RecordReconcileReq,
225 reconcile.RecordResult,
226 ),
227 reconcile.WithFieldOwner(r.Name),
228 reconcile.WithEventRecorder(r.EventRecorder),
229 }...)
230
231 r.Metrics.RecordDuration(ctx, gkeCluster, reconcileStart)
232 r.Metrics.RecordReadiness(ctx, gkeCluster)
233 deleteResourceEntry(gkeCluster)
234 }()
235
236 log = log.WithValues("name", gkeCluster.Spec.Name, "spec", gkeCluster.Spec, "gke reconciler concurrency", r.Concurrency)
237 ctx = logr.NewContext(ctx, log)
238 log.Info("reconciling started for GKECluster")
239
240
241 if err := gkeCluster.Spec.Valid(); err != nil {
242 log.Error(err, ErrInvalidGKECluster)
243 recErr = recerr.NewStalled(fmt.Errorf("invalid spec: %w", err), gkeClusterApi.InvalidSpecReason)
244 return
245 }
246
247 if err := reconcile.Progressing(ctx, gkeCluster, patcher, r.PatchOpts()...); err != nil {
248 recErr = recerr.New(err, gkeClusterApi.ReconcileFailedReason)
249 return
250 }
251
252 var unstructuredObjs []*unstructured.Unstructured
253 uobj, recErr := r.createContainerCluster(ctx, gkeCluster)
254 if recErr != nil {
255 return
256 }
257 unstructuredObjs = append(unstructuredObjs, uobj)
258
259 uobj, recErr = r.createContainerNodePool(ctx, gkeCluster)
260 if recErr != nil {
261 return
262 }
263 unstructuredObjs = append(unstructuredObjs, uobj)
264
265 r.ResourceManager.SetOwnerLabels(unstructuredObjs, r.Name, "")
266
267 changeSet, err := r.ResourceManager.ApplyAll(ctx, unstructuredObjs, sap.ApplyOptions{
268 Force: false,
269 WaitTimeout: r.WaitForSetTimeout,
270 })
271 if err != nil {
272 recErr = recerr.New(fmt.Errorf("failed to apply resources: %w", err), gkeClusterApi.ApplyFailedReason)
273 return
274 }
275
276 containerCluster, recErr := r.getContainerClusterWhenReady(ctx, gkeCluster)
277 if recErr != nil {
278 return
279 }
280 if containerCluster == nil {
281 recErr = recerr.NewWait(errors.New(ErrContainerClusterNotReady), gkeClusterApi.ContainerClusterNotReadyReason, r.DefaultRequeue)
282 return
283 }
284
285 clusterClient, err := r.getGKEClient(ctx, gkeCluster.Name, containerCluster)
286 if err != nil {
287 recErr = recerr.New(fmt.Errorf("failed to create client for container cluster: %w", err), gkeClusterApi.EdgeBootstrapFailedReason)
288 return
289 }
290
291 totpToken, err := totp.GenerateTotp(r.TotpSecret)
292 if err != nil {
293 log.Error(err, "unable to create totp token from cluster id")
294 recErr = recerr.New(fmt.Errorf("unable to create totp token from cluster id: %w", err), gkeClusterApi.EdgeBootstrapFailedReason)
295 return
296 }
297
298 err = r.bootstrapCluster(ctx, clusterClient, gkeCluster, totpToken.Code)
299 if err != nil {
300 recErr = recerr.New(fmt.Errorf("failed to bootstrap cluster: %w", err), gkeClusterApi.EdgeBootstrapFailedReason)
301 return
302 }
303
304 if err = bs.CleanUpKustomizations(ctx, gkeCluster.Name, constants.EdgeBucketName, "latest", log, clusterClient); err != nil {
305 recErr = recerr.New(fmt.Errorf("failed to apply kustomizations: %w", err), gkeClusterApi.EdgeBootstrapFailedReason)
306 return
307 }
308
309 if gkeCluster.Spec.Fleet == fleet.Cluster {
310 hash := uuid.FromUUID(gkeCluster.ObjectMeta.Name).Hash()
311 clusterCtlSAName := fmt.Sprintf("cctl-%s", hash)
312 syncedObjectCtlSAName := fmt.Sprintf("soctl-%s", hash)
313
314 clusterctlSA := &corev1.ServiceAccount{
315 TypeMeta: metav1.TypeMeta{
316 Kind: "ServiceAccount",
317 },
318 ObjectMeta: metav1.ObjectMeta{
319 Name: "clusterctl",
320 Namespace: "clusterctl",
321 Labels: map[string]string{
322 constants.PlatformComponent: "clusterctl",
323 },
324 Annotations: map[string]string{
325 "iam.gke.io/gcp-service-account": iam.SvcAccountEmail(clusterCtlSAName, gkeCluster.Spec.ProjectID),
326 },
327 },
328 }
329 clusterctlSACopy := clusterctlSA.DeepCopy()
330 if _, err = controllerutil.CreateOrUpdate(ctx, clusterClient, clusterctlSACopy, func() error {
331 clusterctlSACopy.Annotations = clusterctlSA.Annotations
332 return nil
333 }); err != nil {
334 recErr = recerr.New(err, gkeClusterApi.ServiceAccountCreationFailedReason)
335 return
336 }
337
338 syncedobjectctlSA := &corev1.ServiceAccount{
339 TypeMeta: metav1.TypeMeta{
340 Kind: "ServiceAccount",
341 },
342 ObjectMeta: metav1.ObjectMeta{
343 Name: "syncedobjectctl",
344 Namespace: "syncedobjectctl",
345 Labels: map[string]string{
346 constants.PlatformComponent: "syncedobjectctl",
347 },
348 Annotations: map[string]string{
349 "iam.gke.io/gcp-service-account": iam.SvcAccountEmail(syncedObjectCtlSAName, gkeCluster.Spec.ProjectID),
350 },
351 },
352 }
353 syncedobjectctlSACopy := syncedobjectctlSA.DeepCopy()
354 if _, err = controllerutil.CreateOrUpdate(ctx, clusterClient, syncedobjectctlSACopy, func() error {
355 syncedobjectctlSACopy.Annotations = syncedobjectctlSA.Annotations
356 return nil
357 }); err != nil {
358 recErr = recerr.New(err, gkeClusterApi.ServiceAccountCreationFailedReason)
359 return
360 }
361 }
362
363 gkeCluster.Status.Inventory = inventory.New(inventory.FromSapChangeSet(changeSet))
364 if oldStatus.Inventory != nil {
365 diff, err := inventory.Diff(oldStatus.Inventory, gkeCluster.GetInventory())
366 if err != nil {
367 recErr = recerr.New(err, gkeClusterApi.PruneFailedReason)
368 return
369 }
370 if len(diff) > 0 {
371 log.Info("inventory diff", diff)
372 prune, err := ff.FeatureEnabledForContext(ff.NewClusterContext(gkeCluster.Name), ff.UseClusterCTLPruning, true)
373 if err != nil {
374 log.Error(err, "unable to get ld flag for prunning, defaulting to prune enabled")
375 }
376 if prune {
377 changeSet, err := r.ResourceManager.DeleteAll(ctx, diff, sap.DefaultDeleteOptions())
378 if err != nil {
379 recErr = recerr.New(err, gkeClusterApi.PruneFailedReason)
380 return
381 }
382 log.Info("pruned objects", "changeset", changeSet)
383 }
384 }
385 }
386 log.Info("GKECluster reconciled successfully")
387
388 conditions.MarkTrue(gkeCluster, status.ReadyCondition, gkeClusterApi.GKEClusterReadyReason, "GKECluster reconciled successfully")
389 result = reconcile.ResultSuccess
390 return
391 }
392
393 func (r *GKEClusterReconciler) createContainerCluster(ctx context.Context, gkeCluster *gkeClusterApi.GKECluster) (*unstructured.Unstructured, recerr.Error) {
394 log := ctrl.LoggerFrom(ctx).WithName("create-container-cluster")
395 containerClusterKey := gkeCluster.ContainerClusterKey()
396
397 cc := k8objectsutils.BuildContainerCluster(gkeCluster, containerClusterKey)
398 cc.ObjectMeta.OwnerReferences = gkeClusterApi.OwnerReference(gkeCluster)
399
400 existingCC := &containerAPI.ContainerCluster{}
401 err := r.Client.Get(ctx, containerClusterKey, existingCC)
402 if client.IgnoreNotFound(err) != nil {
403 log.Error(err, "failed to get GKECluster")
404 return nil, recerr.New(err, gkeClusterApi.ContainerClusterCreationFailedReason)
405 } else if kerrors.IsNotFound(err) {
406
407
408 cc = r.setNetworkConfig(ctx, gkeCluster, cc)
409 } else {
410
411 mapExistingImmutableFieldsCC(cc, existingCC)
412 }
413
414 uobj, err := unstructuredutil.ToUnstructured(cc)
415 if err != nil {
416 return uobj, recerr.New(fmt.Errorf(ErrToUnstructured, cc.Kind, cc.Namespace, cc.Name, err), gkeClusterApi.ApplyFailedReason)
417 }
418
419 return uobj, nil
420 }
421
422 func (r *GKEClusterReconciler) setNetworkConfig(ctx context.Context, gkeCluster *gkeClusterApi.GKECluster, cc *containerAPI.ContainerCluster) *containerAPI.ContainerCluster {
423 log := ctrl.LoggerFrom(ctx).WithName("set-network-config")
424
425 if gkeCluster.Spec.Fleet == fleet.Cluster {
426 netcfg, err := r.IPRangerClient.GetNetcfg(gkeCluster.Spec.ProjectID, gkeCluster.Spec.Location, gkeCluster.Spec.Name)
427 if err != nil {
428 log.Error(err, ErrCreatingContainerCluster, gkeClusterApi.ContainerClusterCreationFailedReason, "unable to get network spec from ipranger, using gke defaults")
429 } else {
430 cc = k8objectsutils.WithNetworkConfig(cc, netcfg.Network, netcfg.Subnetwork, netcfg.Netmask)
431 }
432 }
433 return cc
434 }
435
436
437
438 func mapExistingImmutableFieldsCC(cc *containerAPI.ContainerCluster, existingCC *containerAPI.ContainerCluster) {
439 cc.Spec.NetworkRef = existingCC.Spec.NetworkRef
440 cc.Spec.SubnetworkRef = existingCC.Spec.SubnetworkRef
441 cc.Spec.Location = existingCC.Spec.Location
442 cc.Spec.IpAllocationPolicy = existingCC.Spec.IpAllocationPolicy
443 cc.Spec.InitialNodeCount = existingCC.Spec.InitialNodeCount
444 cc.Spec.ClusterAutoscaling = existingCC.Spec.ClusterAutoscaling
445 cc.Spec.ClusterIpv4Cidr = existingCC.Spec.ClusterIpv4Cidr
446 cc.Spec.ConfidentialNodes = existingCC.Spec.ConfidentialNodes
447 cc.Spec.DefaultMaxPodsPerNode = existingCC.Spec.DefaultMaxPodsPerNode
448 cc.Spec.Description = existingCC.Spec.Description
449 cc.Spec.EnableAutopilot = existingCC.Spec.EnableAutopilot
450 cc.Spec.EnableKubernetesAlpha = existingCC.Spec.EnableKubernetesAlpha
451 cc.Spec.EnableTpu = existingCC.Spec.EnableTpu
452 cc.Spec.Description = existingCC.Spec.Description
453 cc.Spec.Description = existingCC.Spec.Description
454 cc.Spec.MasterAuth = existingCC.Spec.MasterAuth
455 cc.Spec.NetworkingMode = existingCC.Spec.NetworkingMode
456 cc.Spec.NodeConfig = existingCC.Spec.NodeConfig
457 cc.Spec.PrivateClusterConfig = existingCC.Spec.PrivateClusterConfig
458 cc.Spec.ResourceID = existingCC.Spec.ResourceID
459 }
460
461 func (r *GKEClusterReconciler) createContainerNodePool(ctx context.Context, gkeCluster *gkeClusterApi.GKECluster) (*unstructured.Unstructured, recerr.Error) {
462 log := ctrl.LoggerFrom(ctx).WithName("create-container-node-pool")
463 containerClusterKey := gkeCluster.ContainerClusterKey()
464
465 clusterConfig := clusterConfigs[gkeCluster.Spec.Fleet.String()]
466
467 nodePool := k8objectsutils.BuildContainerNodePool(
468 gkeCluster.Spec.ProjectID,
469 gkeCluster.Spec.Banner,
470 gkeCluster.Spec.Organization,
471 containerClusterKey.Namespace,
472 containerClusterKey.Name,
473 gkeCluster.Spec.Location,
474 clusterConfig.MachineType,
475 clusterConfig.NumNode,
476 clusterConfig.MinNodes,
477 clusterConfig.MaxNodes,
478 string(gkeCluster.Spec.Fleet),
479 clusterConfig.Autoscale,
480 )
481
482 nodePool, err := r.handleExistingNodePool(ctx, containerClusterKey, log, clusterConfig, nodePool)
483 if err != nil {
484 return nil, recerr.New(err, gkeClusterApi.ContainerNodePoolCreationFailedReason)
485 }
486
487 nodePool.ObjectMeta.OwnerReferences = gkeClusterApi.OwnerReference(gkeCluster)
488 uobj, err := unstructuredutil.ToUnstructured(nodePool)
489 if err != nil {
490 return uobj, recerr.New(fmt.Errorf(ErrToUnstructured, nodePool.Kind, nodePool.Namespace, nodePool.Name, err), gkeClusterApi.ApplyFailedReason)
491 }
492 return uobj, nil
493 }
494
495 func (r *GKEClusterReconciler) handleExistingNodePool(ctx context.Context, containerClusterKey client.ObjectKey, log logr.Logger, clusterConfig EdgeClusterConfig, nodePool *containerAPI.ContainerNodePool) (*containerAPI.ContainerNodePool, error) {
496 existingNP := &containerAPI.ContainerNodePool{}
497 err := r.Client.Get(ctx, containerClusterKey, existingNP)
498 if client.IgnoreNotFound(err) != nil {
499 log.Error(err, "failed to get Node pool")
500 return nodePool, err
501 } else if err == nil {
502 if nodePoolConfigChanged(clusterConfig, existingNP) {
503
504 err := r.RemoveExistingNodePool(ctx, existingNP)
505 if err != nil {
506 return nodePool, err
507 }
508 } else {
509 nodePool = mapExistingImmutableFieldsNodePool(nodePool, existingNP)
510 }
511 }
512 return nodePool, nil
513 }
514
515 func (r *GKEClusterReconciler) RemoveExistingNodePool(ctx context.Context, existingNP *containerAPI.ContainerNodePool) error {
516 delete(existingNP.Annotations, meta.DeletionPolicyAnnotation)
517 err := r.Client.Update(ctx, existingNP)
518 if err != nil {
519 return err
520 }
521 err = r.Client.Delete(ctx, existingNP)
522 if err != nil {
523 return err
524 }
525 return nil
526 }
527
528 func nodePoolConfigChanged(newPool EdgeClusterConfig, np *containerAPI.ContainerNodePool) bool {
529 if np.Spec.NodeConfig == nil || np.Spec.NodeConfig.MachineType == nil || *np.Spec.NodeConfig.MachineType != newPool.MachineType {
530 return true
531 }
532
533 if autoScaleChanged(newPool, np) {
534 return true
535 }
536 return false
537 }
538
539 func autoScaleChanged(newPool EdgeClusterConfig, np *containerAPI.ContainerNodePool) bool {
540 if newPool.Autoscale && np.Spec.Autoscaling == nil {
541 return true
542 }
543 if np.Spec.Autoscaling == nil {
544 return false
545 }
546 if np.Spec.Autoscaling.MaxNodeCount == nil || newPool.MaxNodes != *np.Spec.Autoscaling.MaxNodeCount {
547 return true
548 }
549 if np.Spec.Autoscaling.MinNodeCount == nil || newPool.MinNodes != *np.Spec.Autoscaling.MinNodeCount {
550 return true
551 }
552 return false
553 }
554
555 func mapExistingImmutableFieldsNodePool(pool *containerAPI.ContainerNodePool, np *containerAPI.ContainerNodePool) *containerAPI.ContainerNodePool {
556 pool.Spec.InitialNodeCount = np.Spec.InitialNodeCount
557 pool.Spec.Location = np.Spec.Location
558 pool.Spec.MaxPodsPerNode = np.Spec.MaxPodsPerNode
559 pool.Spec.NamePrefix = np.Spec.NamePrefix
560 pool.Spec.PlacementPolicy = np.Spec.PlacementPolicy
561 pool.Spec.NodeConfig = np.Spec.NodeConfig
562 pool.Spec.ResourceID = np.Spec.ResourceID
563 return pool
564 }
565
566 func (r *GKEClusterReconciler) getContainerClusterWhenReady(ctx context.Context, gkeCluster *gkeClusterApi.GKECluster) (*containerAPI.ContainerCluster, recerr.Error) {
567 log := ctrl.LoggerFrom(ctx).WithName("get-container-cluster")
568 containerClusterKey := gkeCluster.ContainerClusterKey()
569
570 cluster := &containerAPI.ContainerCluster{}
571 if err := r.Client.Get(ctx, containerClusterKey, cluster); err != nil {
572 log.Error(err, ErrGettingContainerCluster)
573 return nil, recerr.New(err, gkeClusterApi.ContainerClusterNotReadyReason)
574 }
575
576 if ready, _ := meta.IsReady(cluster.Status.Conditions); !ready {
577 return nil, nil
578 }
579
580 nodePool := &containerAPI.ContainerNodePool{}
581 if err := r.Client.Get(ctx, containerClusterKey, nodePool); err != nil {
582 log.Error(err, ErrGettingContainerNodePool)
583 return nil, recerr.New(err, gkeClusterApi.ContainerNodePoolNotReadyReason)
584 }
585
586 if ready, _ := meta.IsReady(nodePool.Status.Conditions); !ready {
587 return nil, nil
588 }
589
590 return cluster, nil
591 }
592
593 func (r *GKEClusterReconciler) getGKEClient(ctx context.Context, clusterEdgeID string, cc *containerAPI.ContainerCluster) (client.Client, error) {
594 log := ctrl.LoggerFrom(ctx).WithName("get-gke-client")
595 gkeClientCache.Lock()
596 defer gkeClientCache.Unlock()
597
598 clusterClient, ok := gkeClientCache.c[clusterEdgeID]
599 if !ok {
600 var err error
601 clusterClient, err = r.CreateClient(*cc, client.Options{Scheme: r.Scheme})
602 if err != nil {
603 log.Error(err, "failed to create client for container cluster")
604 return nil, err
605 }
606 gkeClientCache.c[clusterEdgeID] = clusterClient
607 }
608 return clusterClient, nil
609 }
610
611 func (r *GKEClusterReconciler) bootstrapCluster(ctx context.Context, cl client.Client, gkeCluster *gkeClusterApi.GKECluster, token string) error {
612 log := ctrl.LoggerFrom(ctx).WithName("bootstrap-cluster")
613
614 reg, err := registration.NewBuilder().
615 Banner(gkeCluster.Spec.Banner).
616 Store(gkeCluster.Spec.Name).
617 ClusterType(clusterConstants.GKE).
618 BSLOrganization(gkeCluster.Spec.Organization).
619 APIEndpoint(r.EdgeAPI).
620 TotpToken(token).
621 ClusterEdgeID(gkeCluster.Name).
622 CreateBSLSite(true).
623 Fleet(string(gkeCluster.Spec.Fleet)).
624 ForceBootstrap(true).
625 BootstrapBuild()
626 if err != nil {
627 log.Error(err, "failed building a registration for edge bootstrap api", gkeClusterApi.EdgeBootstrapFailedReason, ErrEdgeBootstrapAPIFailed)
628 return err
629 }
630 reg.Client = cl
631 err = reg.BootstrapCluster(ctx)
632 if err != nil {
633 log.Error(err, "failed bootstrapping cluster with edge bootstrap api", gkeClusterApi.EdgeBootstrapFailedReason, ErrEdgeBootstrapAPIFailed)
634 return err
635 }
636 return nil
637 }
638
639 func (r *GKEClusterReconciler) setResourceManager() error {
640 if r.ResourceManager == nil {
641 sapMngr, err := sap.NewResourceManagerFromConfig(r.manager.GetConfig(),
642 client.Options{
643 HTTPClient: r.manager.GetHTTPClient(),
644 Mapper: r.RESTMapper(),
645 Scheme: r.Scheme,
646 },
647 sap.Owner{Field: r.Name, Group: GKEClusterLabel})
648 if err != nil {
649 return err
650 }
651 r.ResourceManager = sapMngr
652 }
653 return nil
654 }
655
View as plain text