1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package controller
16
17 import (
18 "context"
19 "strings"
20 "testing"
21
22 "github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/k8s"
23 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/cluster"
24
25 "github.com/ghodss/yaml"
26 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
27 "sigs.k8s.io/controller-runtime/pkg/client"
28 "sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/declarative/pkg/manifest"
29 )
30
31 var NamespacedComponentsTemplate = []string{`
32 apiVersion: v1
33 kind: ServiceAccount
34 metadata:
35 annotations:
36 iam.gke.io/gcp-service-account: ${SERVICE_ACCOUNT?}
37 labels:
38 cnrm.cloud.google.com/scoped-namespace: ${NAMESPACE?}
39 cnrm.cloud.google.com/system: "true"
40 name: cnrm-controller-manager-${NAMESPACE?}
41 namespace: cnrm-system
42 `, `
43 apiVersion: rbac.authorization.k8s.io/v1
44 kind: RoleBinding
45 metadata:
46 labels:
47 cnrm.cloud.google.com/scoped-namespace: ${NAMESPACE?}
48 cnrm.cloud.google.com/system: "true"
49 name: cnrm-admin-binding-${NAMESPACE?}
50 namespace: ${NAMESPACE?}
51 roleRef:
52 apiGroup: rbac.authorization.k8s.io
53 kind: ClusterRole
54 name: cnrm-admin
55 subjects:
56 - kind: ServiceAccount
57 name: cnrm-controller-manager-${NAMESPACE?}
58 namespace: cnrm-system
59 `, `
60 apiVersion: v1
61 kind: Service
62 metadata:
63 labels:
64 cnrm.cloud.google.com/monitored: "true"
65 cnrm.cloud.google.com/scoped-namespace: ${NAMESPACE?}
66 cnrm.cloud.google.com/system: "true"
67 name: cnrm-manager-${NAMESPACE?}
68 namespace: cnrm-system
69 spec:
70 ports:
71 - name: controller-manager
72 port: 443
73 - name: metrics
74 port: 8888
75 selector:
76 cnrm.cloud.google.com/component: cnrm-controller-manager
77 cnrm.cloud.google.com/scoped-namespace: ${NAMESPACE?}
78 cnrm.cloud.google.com/system: "true"
79 `, `
80 apiVersion: apps/v1
81 kind: StatefulSet
82 metadata:
83 labels:
84 cnrm.cloud.google.com/component: cnrm-controller-manager
85 cnrm.cloud.google.com/scoped-namespace: ${NAMESPACE?}
86 cnrm.cloud.google.com/system: "true"
87 name: cnrm-controller-manager-${NAMESPACE?}
88 namespace: cnrm-system
89 spec:
90 selector:
91 matchLabels:
92 cnrm.cloud.google.com/component: cnrm-controller-manager
93 cnrm.cloud.google.com/scoped-namespace: ${NAMESPACE?}
94 cnrm.cloud.google.com/system: "true"
95 serviceName: cnrm-manager-${NAMESPACE?}
96 template:
97 metadata:
98 labels:
99 cnrm.cloud.google.com/component: cnrm-controller-manager
100 cnrm.cloud.google.com/scoped-namespace: ${NAMESPACE?}
101 cnrm.cloud.google.com/system: "true"
102 spec:
103 containers:
104 - args: ["--scoped-namespace=${NAMESPACE?}", "--stderrthreshold=INFO", "--prometheus-scrape-endpoint=:8888"]
105 command: ["/configconnector/manager"]
106 image: gcr.io/gke-release/cnrm/controller:4af93f1
107 name: manager
108 - command: ["/monitor", "--source=configconnector:http://localhost:8888?whitelisted=reconcile_requests_total,reconcile_request_duration_seconds,reconcile_workers_total,reconcile_occupied_workers_total,internal_errors_total&customResourceType=k8s_container&customLabels[container_name]&customLabels[project_id]&customLabels[location]&customLabels[cluster_name]&customLabels[namespace_name]&customLabels[pod_name]", "--stackdriver-prefix=kubernetes.io/internal/addons"]
109 image: gke.gcr.io/prometheus-to-sd:v0.9.1
110 name: prom-to-sd
111 `}
112
113 var FooCRD = `
114 apiVersion: apiextensions.k8s.io/v1
115 kind: CustomResourceDefinition
116 metadata:
117 name: foos.test.cnrm.cloud.google.com
118 labels:
119 cnrm.cloud.google.com/system: "true"
120 cnrm.cloud.google.com/managed-by-kcc: "true"
121 spec:
122 scope: Namespaced
123 group: test.cnrm.cloud.google.com
124 names:
125 kind: foo
126 plural: foos
127 versions:
128 - name: v1alpha1
129 schema:
130 openAPIV3Schema:
131 type: object
132 served: true
133 storage: true
134 `
135
136 var barCRD = `
137 apiVersion: apiextensions.k8s.io/v1
138 kind: CustomResourceDefinition
139 metadata:
140 name: bars.test.cnrm.cloud.google.com
141 labels:
142 cnrm.cloud.google.com/system: "true"
143 cnrm.cloud.google.com/managed-by-kcc: "true"
144 spec:
145 scope: Namespaced
146 group: test.cnrm.cloud.google.com
147 names:
148 kind: bar
149 plural: bars
150 versions:
151 - name: v1beta1
152 schema:
153 openAPIV3Schema:
154 type: object
155 served: true
156 storage: true
157 `
158
159 var nonKCCCRD = `
160 apiVersion: apiextensions.k8s.io/v1
161 kind: CustomResourceDefinition
162 metadata:
163 name: bars.test.nonkcc.cloud.google.com
164 labels:
165 cnrm.cloud.google.com/system: "true"
166 cnrm.cloud.google.com/managed-by-kcc: "true"
167 spec:
168 scope: Namespaced
169 group: test.nonkcc.cloud.google.com
170 names:
171 kind: bar
172 plural: bars
173 versions:
174 - name: v1beta1
175 schema:
176 openAPIV3Schema:
177 type: object
178 served: true
179 storage: true
180 `
181
182 var defectiveCRD = `
183 apiVersion: apiextensions.k8s.io/v1
184 kind: CustomResourceDefinition
185 metadata:
186 name: bars.test.nonkcc.cloud.google.com
187 labels:
188 cnrm.cloud.google.com/system: "true"
189 cnrm.cloud.google.com/managed-by-kcc: "true"
190 spec:
191 scope: Namespaced
192 group: test.nonkcc.cloud.google.com
193 names:
194 kind: bar
195 plural: bars
196 versions: nil
197 `
198
199 var SystemNs = `apiVersion: v1
200 kind: Namespace
201 metadata:
202 name: cnrm-system
203 `
204
205 var ClusterModeOnlyWorkloadIdentityComponents = []string{`
206 apiVersion: v1
207 kind: ServiceAccount
208 metadata:
209 annotations:
210 iam.gke.io/gcp-service-account: ${SERVICE_ACCOUNT?}
211 name: cnrm-controller-manager
212 namespace: cnrm-system
213 `, `
214 apiVersion: v1
215 kind: Service
216 metadata:
217 name: cnrm-manager
218 namespace: cnrm-system
219 spec:
220 ports:
221 - name: controller-manager
222 port: 443
223 - name: metrics
224 port: 8888
225 selector:
226 cnrm.cloud.google.com/component: cnrm-controller-manager
227 cnrm.cloud.google.com/system: "true"
228 `, `
229 apiVersion: apps/v1
230 kind: StatefulSet
231 metadata:
232 labels:
233 cnrm.cloud.google.com/component: cnrm-controller-manager
234 cnrm.cloud.google.com/system: "true"
235 name: cnrm-controller-manager
236 namespace: cnrm-system
237 spec:
238 selector:
239 matchLabels:
240 cnrm.cloud.google.com/component: cnrm-controller-manager
241 cnrm.cloud.google.com/system: "true"
242 serviceName: cnrm-manager
243 template:
244 metadata:
245 labels:
246 cnrm.cloud.google.com/component: cnrm-controller-manager
247 cnrm.cloud.google.com/system: "true"
248 `}
249
250 var ClusterModeOnlyGCPComponents = []string{`
251 apiVersion: v1
252 kind: ServiceAccount
253 metadata:
254 name: cnrm-controller-manager
255 namespace: cnrm-system
256 `, `
257 apiVersion: v1
258 kind: Service
259 metadata:
260 name: cnrm-manager
261 namespace: cnrm-system
262 spec:
263 ports:
264 - name: controller-manager
265 port: 443
266 - name: metrics
267 port: 8888
268 selector:
269 cnrm.cloud.google.com/component: cnrm-controller-manager
270 cnrm.cloud.google.com/system: "true"
271 `, `
272 apiVersion: apps/v1
273 kind: StatefulSet
274 metadata:
275 labels:
276 cnrm.cloud.google.com/component: cnrm-controller-manager
277 cnrm.cloud.google.com/system: "true"
278 name: cnrm-controller-manager
279 namespace: cnrm-system
280 spec:
281 selector:
282 matchLabels:
283 cnrm.cloud.google.com/component: cnrm-controller-manager
284 cnrm.cloud.google.com/system: "true"
285 serviceName: cnrm-manager
286 template:
287 metadata:
288 labels:
289 cnrm.cloud.google.com/component: cnrm-controller-manager
290 cnrm.cloud.google.com/system: "true"
291 spec:
292 volumes:
293 - name: gcp-service-account
294 secret:
295 secretName: gcp-key
296 `}
297
298 var PerNamespaceControllerManagerPod = `apiVersion: v1
299 kind: Pod
300 metadata:
301 labels:
302 cnrm.cloud.google.com/component: cnrm-controller-manager
303 cnrm.cloud.google.com/system: "true"
304 name: cnrm-controller-manager-12345-0
305 namespace: cnrm-system
306 spec:
307 containers:
308 - name: manager
309 image: test-image
310 `
311
312 func GetSharedComponentsManifest() []string {
313 res := make([]string, 0)
314 res = append(res, FooCRD, SystemNs)
315 return res
316 }
317
318 func GetManifestsWithAlphaAndBetaCRDs() []string {
319 res := make([]string, 0)
320 res = append(res, FooCRD, barCRD, SystemNs)
321 return res
322 }
323
324 func GetManifestsWithAlphaCRD() []string {
325 res := make([]string, 0)
326 res = append(res, FooCRD, SystemNs)
327 return res
328 }
329
330 func GetManifestsWithBetaCRD() []string {
331 res := make([]string, 0)
332 res = append(res, barCRD, SystemNs)
333 return res
334 }
335
336 func GetManifestsWithNoCRD() []string {
337 res := make([]string, 0)
338 res = append(res, SystemNs)
339 return res
340 }
341
342 func GetManifestsWithNonKCCCRD() []string {
343 res := make([]string, 0)
344 res = append(res, nonKCCCRD, FooCRD, SystemNs)
345 return res
346 }
347
348 func GetManifestsWithDefectiveCRD() []string {
349 res := make([]string, 0)
350 res = append(res, defectiveCRD, SystemNs)
351 return res
352 }
353
354 func GetClusterModeGCPManifest() []string {
355 res := make([]string, 0)
356 res = append(res, GetSharedComponentsManifest()...)
357 res = append(res, ClusterModeOnlyGCPComponents...)
358 return res
359 }
360
361 func GetClusterModeWorkloadIdentityManifest() []string {
362 res := make([]string, 0)
363 res = append(res, GetSharedComponentsManifest()...)
364 res = append(res, ClusterModeOnlyWorkloadIdentityComponents...)
365 return res
366 }
367
368 func GetPerNamespaceManifest() []string {
369 res := make([]string, 0)
370 res = append(res, NamespacedComponentsTemplate...)
371 return res
372 }
373
374 func ManuallyReplaceGSA(components []string, saName string) []string {
375 res := make([]string, 0)
376 for _, s := range components {
377 s = strings.ReplaceAll(s, "${SERVICE_ACCOUNT?}", saName)
378 res = append(res, s)
379 }
380 return res
381 }
382
383 func ManuallyReplaceSecretVolume(components []string, secretName string) []string {
384 res := make([]string, 0)
385 for _, s := range components {
386 s = strings.ReplaceAll(s, "gcp-key", secretName)
387 res = append(res, s)
388 }
389 return res
390 }
391
392 func ManuallyModifyNamespaceTemplates(t *testing.T, template []string, nsName, saName string, userProjectOverride bool, billingProject string, c client.Client) []string {
393 var res []string
394 nsId, err := cluster.GetNamespaceID(k8s.OperatorNamespaceIDConfigMapNN, c, context.TODO(), nsName)
395 if err != nil {
396 t.Fatalf("error getting the id for namespace %v", err)
397 }
398 for _, s := range template {
399 applied := s
400 if strings.Contains(s, "kind: StatefulSet") {
401 if billingProject != "" {
402 applied = strings.ReplaceAll(applied,
403 `args: ["--scoped-namespace=${NAMESPACE?}", "--stderrthreshold=INFO", "--prometheus-scrape-endpoint=:8888"`,
404 `args: ["--scoped-namespace=${NAMESPACE?}", "--stderrthreshold=INFO", "--prometheus-scrape-endpoint=:8888", "--billing-project=`+billingProject+`"`,
405 )
406 }
407
408 if userProjectOverride {
409 applied = strings.ReplaceAll(applied,
410 `args: ["--scoped-namespace=${NAMESPACE?}", "--stderrthreshold=INFO", "--prometheus-scrape-endpoint=:8888"`,
411 `args: ["--scoped-namespace=${NAMESPACE?}", "--stderrthreshold=INFO", "--prometheus-scrape-endpoint=:8888", "--user-project-override=true"`,
412 )
413 }
414
415 applied = strings.ReplaceAll(applied, "cnrm-controller-manager-${NAMESPACE?}", "cnrm-controller-manager-"+nsId)
416 applied = strings.ReplaceAll(applied, k8s.NamespacedManagerServiceTmpl, k8s.NamespacedManagerServicePrefix+nsId)
417 }
418 if strings.Contains(s, "name: cnrm-manager-${NAMESPACE?}") {
419 applied = strings.ReplaceAll(applied, k8s.NamespacedManagerServiceTmpl, k8s.NamespacedManagerServicePrefix+nsId)
420 }
421 applied = strings.ReplaceAll(applied, "${SERVICE_ACCOUNT?}", saName)
422 applied = strings.ReplaceAll(applied, "${NAMESPACE?}", nsName)
423 u := ToUnstructured(t, applied)
424 labels := u.GetLabels()
425 labels[k8s.ConfigConnectorContextNamespaceLabel] = nsName
426 u.SetLabels(labels)
427 applied = ToString(t, u)
428 res = append(res, applied)
429 }
430 return res
431 }
432
433 func ToUnstructured(t *testing.T, objStr string) *unstructured.Unstructured {
434 u := &unstructured.Unstructured{}
435 b := []byte(objStr)
436 err := yaml.Unmarshal(b, u)
437 if err != nil {
438 t.Fatalf("error unmarshalling bytes to unstruct: %v", err)
439 }
440 return u
441 }
442
443 func ToString(t *testing.T, u *unstructured.Unstructured) string {
444 json, err := u.MarshalJSON()
445 if err != nil {
446 t.Fatalf("error marshalling unstructured to json: %v", err)
447 }
448 y, err := yaml.JSONToYAML(json)
449 if err != nil {
450 t.Fatalf("error converting json to yaml: %v", err)
451 }
452 return string(y)
453 }
454
455 func ParseObjects(t *testing.T, ctx context.Context, objects []string) *manifest.Objects {
456 objs := strings.Join(objects, "---\n")
457 m, err := manifest.ParseObjects(ctx, objs)
458 if err != nil {
459 t.Fatalf("while parsing objects: %v", err)
460 }
461 return m
462 }
463
View as plain text