1 package clusterctl
2
3 import (
4 "fmt"
5
6 containerAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/container/v1beta1"
7 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/k8s/v1alpha1"
8 "github.com/google/uuid"
9 corev1 "k8s.io/api/core/v1"
10 "k8s.io/apimachinery/pkg/api/errors"
11 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
12 "sigs.k8s.io/controller-runtime/pkg/client"
13
14 gkeClusterApi "edge-infra.dev/pkg/edge/apis/gkecluster/v1alpha1"
15 "edge-infra.dev/pkg/edge/constants/api/fleet"
16 "edge-infra.dev/pkg/edge/k8objectsutils"
17 "edge-infra.dev/pkg/k8s/konfigkonnector/apis/meta"
18 "edge-infra.dev/pkg/lib/gcp/iam"
19 edgeUUID "edge-infra.dev/pkg/lib/uuid"
20 "edge-infra.dev/test/framework/integration"
21 )
22
23 func (s *Suite) TestGKEClusterReconciler() {
24 name := uuid.New().String()
25 gkeCluster := gkeClusterApi.New(s.ProjectID,
26 s.Banner.Name,
27 s.Organization,
28 name,
29 s.Location,
30 s.NodeVersion,
31 s.NumNodes,
32 fleet.Store, name)
33
34 key := gkeCluster.ContainerClusterKey()
35 s.NoError(s.Client.Create(s.ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: key.Namespace}}))
36 s.NoError(s.Client.Create(s.ctx, gkeCluster))
37
38
39 s.Eventually(func() bool {
40 err := s.Client.Get(s.ctx, client.ObjectKeyFromObject(gkeCluster), gkeCluster)
41 return !errors.IsNotFound(err)
42 }, s.timeout, s.tick, "expected GKECluster was never created")
43
44
45 cluster := &containerAPI.ContainerCluster{}
46 s.Eventually(func() bool {
47 err := s.Client.Get(s.ctx, key, cluster)
48 return !errors.IsNotFound(err)
49 }, s.timeout, s.tick, "expected ContainerCluster was never created")
50 s.Equal(meta.DeletionPolicyAbandon, cluster.Annotations[meta.DeletionPolicyAnnotation])
51
52
53 nodePool := &containerAPI.ContainerNodePool{}
54 s.Eventually(func() bool {
55 err := s.Client.Get(s.ctx, key, nodePool)
56 return !errors.IsNotFound(err)
57 }, s.timeout, s.tick, "expected ContainerNodePool was never created")
58 s.Equal(meta.DeletionPolicyAbandon, nodePool.Annotations[meta.DeletionPolicyAnnotation])
59 s.Equal(clusterConfigs["store"].MachineType, *nodePool.Spec.NodeConfig.MachineType)
60
61 if !integration.IsIntegrationTest() {
62 cluster.Status = containerAPI.ContainerClusterStatus{Conditions: []v1alpha1.Condition{{Status: "True", Type: "Ready"}}}
63 s.NoError(s.Client.Update(s.ctx, cluster))
64
65 nodePool.Status = containerAPI.ContainerNodePoolStatus{Conditions: []v1alpha1.Condition{{Status: "True", Type: "Ready"}}}
66 s.NoError(s.Client.Update(s.ctx, nodePool))
67 }
68
69 s.Eventually(func() bool {
70 s.NoError(s.Client.Get(s.ctx, client.ObjectKeyFromObject(gkeCluster), gkeCluster))
71 return gkeCluster.Status.Inventory != nil && len(gkeCluster.Status.Inventory.Entries) == 2
72 }, s.timeout, s.tick, "expected GKECluster inventory was never created")
73
74 s.NoError(s.Client.Delete(s.ctx, gkeCluster))
75 }
76
77 func (s *Suite) TestGKEClusterImmutableFields() {
78 integration.SkipIf(s.Framework)
79 name := uuid.New().String()
80 gkeCluster := gkeClusterApi.New(s.ProjectID,
81 s.Banner.Name,
82 s.Organization,
83 name,
84 s.Location,
85 s.NodeVersion,
86 s.NumNodes,
87 fleet.Store, name)
88
89 key := gkeCluster.ContainerClusterKey()
90 cc := k8objectsutils.BuildContainerCluster(gkeCluster, key)
91 desc := "tester desc"
92 cc.Spec.Description = &desc
93 cc.Spec.ReleaseChannel = &containerAPI.ClusterReleaseChannel{Channel: "BETA"}
94 s.NoError(s.Client.Create(s.ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: key.Namespace}}))
95 s.NoError(s.Client.Create(s.ctx, cc))
96 cluster := &containerAPI.ContainerCluster{}
97 s.Eventually(func() bool {
98 err := s.Client.Get(s.ctx, key, cluster)
99 return !errors.IsNotFound(err)
100 }, s.timeout, s.tick, "expected ContainerCluster was never created")
101 s.Equal(cluster.Spec.ReleaseChannel.Channel, "BETA")
102 s.Equal(*cluster.Spec.Description, "tester desc")
103 s.NoError(s.Client.Create(s.ctx, gkeCluster))
104
105
106 s.Eventually(func() bool {
107 err := s.Client.Get(s.ctx, client.ObjectKeyFromObject(gkeCluster), gkeCluster)
108 return !errors.IsNotFound(err)
109 }, s.timeout, s.tick, "expected GKECluster was never created")
110
111
112 s.Eventually(func() bool {
113 err := s.Client.Get(s.ctx, key, cluster)
114 if err != nil {
115 return false
116 }
117 if cluster.Spec.ReleaseChannel.Channel != "STABLE" {
118 return false
119 }
120 if cluster.Spec.Description == nil || *cluster.Spec.Description != "tester desc" {
121 return false
122 }
123 return true
124 }, s.timeout, s.tick, "expected ContainerCluster was never created")
125 }
126
127 func (s *Suite) TestGKEClusterReconciler_FleetClusterInfra() {
128 name := uuid.New().String()
129
130 gkeCluster := gkeClusterApi.New(s.ProjectID,
131 s.Banner.Name,
132 s.Organization,
133 name,
134 s.Location,
135 s.NodeVersion,
136 s.NumNodes,
137 fleet.Cluster, name)
138
139 s.testGKEClusterReconciler(gkeCluster)
140 }
141
142 func (s *Suite) TestGKEClusterReconciler_TenantClusterInfra() {
143 name := uuid.New().String()
144
145 gkeCluster := gkeClusterApi.New("tenant-project",
146 s.Banner.Name,
147 s.Organization,
148 name,
149 s.Location,
150 s.NodeVersion,
151 s.NumNodes,
152 fleet.Cluster, name)
153
154 s.testGKEClusterReconciler(gkeCluster)
155 }
156
157 func (s *Suite) testGKEClusterReconciler(gkeCluster *gkeClusterApi.GKECluster) {
158 key := gkeCluster.ContainerClusterKey()
159 s.NoError(s.Client.Create(s.ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: key.Namespace}}))
160 s.NoError(s.Client.Create(s.ctx, gkeCluster))
161
162
163 s.Eventually(func() bool {
164 err := s.Client.Get(s.ctx, client.ObjectKeyFromObject(gkeCluster), gkeCluster)
165 return !errors.IsNotFound(err)
166 }, s.timeout, s.tick, "expected GKECluster was never created")
167
168
169 cluster := &containerAPI.ContainerCluster{}
170 s.Eventually(func() bool {
171 err := s.Client.Get(s.ctx, key, cluster)
172 return !errors.IsNotFound(err)
173 }, s.timeout, s.tick, "expected ContainerCluster was never created")
174
175
176 s.NotNil(cluster.Spec.NetworkRef, "expected networkRef to be set on new GKE ContainerClusters")
177 s.NotNil(cluster.Spec.SubnetworkRef, "expected subnetworkRef to be set on new GKE ContainerClusters")
178 s.NotNil(cluster.Spec.IpAllocationPolicy, "expected ipAllocationPolicy to be set on new GKE ContainerClusters")
179 s.Equal(meta.DeletionPolicyAbandon, cluster.Annotations[meta.DeletionPolicyAnnotation])
180
181
182 s.NotEmpty(cluster.Spec.MasterAuthorizedNetworksConfig.CidrBlocks, "expected a MasterAuthorizedNetworksConfig block")
183
184
185 nodePool := &containerAPI.ContainerNodePool{}
186 s.Eventually(func() bool {
187 err := s.Client.Get(s.ctx, key, nodePool)
188 return !errors.IsNotFound(err)
189 }, s.timeout, s.tick, "expected ContainerNodePool was never created")
190 s.Equal(meta.DeletionPolicyAbandon, nodePool.Annotations[meta.DeletionPolicyAnnotation])
191
192 if !integration.IsIntegrationTest() {
193 cluster.Status = containerAPI.ContainerClusterStatus{Conditions: []v1alpha1.Condition{{Status: "True", Type: "Ready"}}}
194 s.NoError(s.Client.Update(s.ctx, cluster))
195
196 nodePool.Status = containerAPI.ContainerNodePoolStatus{Conditions: []v1alpha1.Condition{{Status: "True", Type: "Ready"}}}
197 s.NoError(s.Client.Update(s.ctx, nodePool))
198 }
199
200 clusterClient, err := s.ClusterClient(*cluster, client.Options{Scheme: s.Scheme})
201 s.NoError(err)
202
203 hash := edgeUUID.FromUUID(gkeCluster.ObjectMeta.Name).Hash()
204 clusterCtlSAName := fmt.Sprintf("cctl-%s", hash)
205 syncedObjectCtlSAName := fmt.Sprintf("soctl-%s", hash)
206
207
208 clusterctlSA := &corev1.ServiceAccount{}
209 s.Eventually(func() bool {
210 err := clusterClient.Get(s.ctx, client.ObjectKey{Namespace: "clusterctl", Name: "clusterctl"}, clusterctlSA)
211 return !errors.IsNotFound(err) && iam.SvcAccountEmail(clusterCtlSAName, gkeCluster.Spec.ProjectID) == clusterctlSA.Annotations["iam.gke.io/gcp-service-account"]
212 }, s.timeout, s.tick, "expected clusterctl service account to be created")
213
214
215 syncedobjectctlSA := &corev1.ServiceAccount{}
216 s.Eventually(func() bool {
217 err := clusterClient.Get(s.ctx, client.ObjectKey{Namespace: "syncedobjectctl", Name: "syncedobjectctl"}, syncedobjectctlSA)
218 return !errors.IsNotFound(err) && iam.SvcAccountEmail(syncedObjectCtlSAName, gkeCluster.Spec.ProjectID) == syncedobjectctlSA.Annotations["iam.gke.io/gcp-service-account"]
219 }, s.timeout, s.tick, "expected syncedobject service account to be created")
220
221 s.NoError(s.Client.Delete(s.ctx, gkeCluster))
222 }
223
View as plain text