...

Source file src/github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/test/controller/manifest.go

Documentation: github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/test/controller

     1  // Copyright 2022 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package controller
    16  
    17  import (
    18  	"context"
    19  	"strings"
    20  	"testing"
    21  
    22  	"github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/k8s"
    23  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/cluster"
    24  
    25  	"github.com/ghodss/yaml"
    26  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    27  	"sigs.k8s.io/controller-runtime/pkg/client"
    28  	"sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/declarative/pkg/manifest"
    29  )
    30  
    31  var NamespacedComponentsTemplate = []string{`
    32  apiVersion: v1
    33  kind: ServiceAccount
    34  metadata:
    35    annotations:
    36      iam.gke.io/gcp-service-account: ${SERVICE_ACCOUNT?}
    37    labels:
    38      cnrm.cloud.google.com/scoped-namespace: ${NAMESPACE?}
    39      cnrm.cloud.google.com/system: "true"
    40    name: cnrm-controller-manager-${NAMESPACE?}
    41    namespace: cnrm-system
    42  `, `
    43  apiVersion: rbac.authorization.k8s.io/v1
    44  kind: RoleBinding
    45  metadata:
    46    labels:
    47      cnrm.cloud.google.com/scoped-namespace: ${NAMESPACE?}
    48      cnrm.cloud.google.com/system: "true"
    49    name: cnrm-admin-binding-${NAMESPACE?}
    50    namespace: ${NAMESPACE?}
    51  roleRef:
    52    apiGroup: rbac.authorization.k8s.io
    53    kind: ClusterRole
    54    name: cnrm-admin
    55  subjects:
    56  - kind: ServiceAccount
    57    name: cnrm-controller-manager-${NAMESPACE?}
    58    namespace: cnrm-system
    59  `, `
    60  apiVersion: v1
    61  kind: Service
    62  metadata:
    63    labels:
    64      cnrm.cloud.google.com/monitored: "true"
    65      cnrm.cloud.google.com/scoped-namespace: ${NAMESPACE?}
    66      cnrm.cloud.google.com/system: "true"
    67    name: cnrm-manager-${NAMESPACE?}
    68    namespace: cnrm-system
    69  spec:
    70    ports:
    71    - name: controller-manager
    72      port: 443
    73    - name: metrics
    74      port: 8888
    75    selector:
    76      cnrm.cloud.google.com/component: cnrm-controller-manager
    77      cnrm.cloud.google.com/scoped-namespace: ${NAMESPACE?}
    78      cnrm.cloud.google.com/system: "true"
    79  `, `
    80  apiVersion: apps/v1
    81  kind: StatefulSet
    82  metadata:
    83    labels:
    84      cnrm.cloud.google.com/component: cnrm-controller-manager
    85      cnrm.cloud.google.com/scoped-namespace: ${NAMESPACE?}
    86      cnrm.cloud.google.com/system: "true"
    87    name: cnrm-controller-manager-${NAMESPACE?}
    88    namespace: cnrm-system
    89  spec:
    90    selector:
    91      matchLabels:
    92        cnrm.cloud.google.com/component: cnrm-controller-manager
    93        cnrm.cloud.google.com/scoped-namespace: ${NAMESPACE?}
    94        cnrm.cloud.google.com/system: "true"
    95    serviceName: cnrm-manager-${NAMESPACE?}
    96    template:
    97      metadata:
    98        labels:
    99          cnrm.cloud.google.com/component: cnrm-controller-manager
   100          cnrm.cloud.google.com/scoped-namespace: ${NAMESPACE?}
   101          cnrm.cloud.google.com/system: "true"
   102      spec:
   103        containers:
   104        - args: ["--scoped-namespace=${NAMESPACE?}", "--stderrthreshold=INFO", "--prometheus-scrape-endpoint=:8888"]
   105          command: ["/configconnector/manager"]
   106          image: gcr.io/gke-release/cnrm/controller:4af93f1
   107          name: manager
   108        - command: ["/monitor", "--source=configconnector:http://localhost:8888?whitelisted=reconcile_requests_total,reconcile_request_duration_seconds,reconcile_workers_total,reconcile_occupied_workers_total,internal_errors_total&customResourceType=k8s_container&customLabels[container_name]&customLabels[project_id]&customLabels[location]&customLabels[cluster_name]&customLabels[namespace_name]&customLabels[pod_name]", "--stackdriver-prefix=kubernetes.io/internal/addons"]
   109          image: gke.gcr.io/prometheus-to-sd:v0.9.1
   110          name: prom-to-sd
   111  `}
   112  
   113  var FooCRD = `
   114  apiVersion: apiextensions.k8s.io/v1
   115  kind: CustomResourceDefinition
   116  metadata:
   117    name: foos.test.cnrm.cloud.google.com
   118    labels:
   119      cnrm.cloud.google.com/system: "true"
   120      cnrm.cloud.google.com/managed-by-kcc: "true"
   121  spec:
   122    scope: Namespaced
   123    group: test.cnrm.cloud.google.com
   124    names:
   125      kind: foo
   126      plural: foos
   127    versions:
   128    - name: v1alpha1
   129      schema:
   130        openAPIV3Schema:
   131          type: object
   132      served: true
   133      storage: true
   134  `
   135  
   136  var barCRD = `
   137  apiVersion: apiextensions.k8s.io/v1
   138  kind: CustomResourceDefinition
   139  metadata:
   140    name: bars.test.cnrm.cloud.google.com
   141    labels:
   142      cnrm.cloud.google.com/system: "true"
   143      cnrm.cloud.google.com/managed-by-kcc: "true"
   144  spec:
   145    scope: Namespaced
   146    group: test.cnrm.cloud.google.com
   147    names:
   148      kind: bar
   149      plural: bars
   150    versions:
   151    - name: v1beta1
   152      schema:
   153        openAPIV3Schema:
   154          type: object
   155      served: true
   156      storage: true
   157  `
   158  
   159  var nonKCCCRD = `
   160  apiVersion: apiextensions.k8s.io/v1
   161  kind: CustomResourceDefinition
   162  metadata:
   163    name: bars.test.nonkcc.cloud.google.com
   164    labels:
   165      cnrm.cloud.google.com/system: "true"
   166      cnrm.cloud.google.com/managed-by-kcc: "true"
   167  spec:
   168    scope: Namespaced
   169    group: test.nonkcc.cloud.google.com
   170    names:
   171      kind: bar
   172      plural: bars
   173    versions:
   174    - name: v1beta1
   175      schema:
   176        openAPIV3Schema:
   177          type: object
   178      served: true
   179      storage: true
   180  `
   181  
   182  var defectiveCRD = `
   183  apiVersion: apiextensions.k8s.io/v1
   184  kind: CustomResourceDefinition
   185  metadata:
   186    name: bars.test.nonkcc.cloud.google.com
   187    labels:
   188      cnrm.cloud.google.com/system: "true"
   189      cnrm.cloud.google.com/managed-by-kcc: "true"
   190  spec:
   191    scope: Namespaced
   192    group: test.nonkcc.cloud.google.com
   193    names:
   194      kind: bar
   195      plural: bars
   196    versions: nil
   197  `
   198  
   199  var SystemNs = `apiVersion: v1
   200  kind: Namespace
   201  metadata:
   202    name: cnrm-system
   203  `
   204  
   205  var ClusterModeOnlyWorkloadIdentityComponents = []string{`
   206  apiVersion: v1
   207  kind: ServiceAccount
   208  metadata:
   209    annotations:
   210      iam.gke.io/gcp-service-account: ${SERVICE_ACCOUNT?}
   211    name: cnrm-controller-manager
   212    namespace: cnrm-system
   213  `, `
   214  apiVersion: v1
   215  kind: Service
   216  metadata:
   217    name: cnrm-manager
   218    namespace: cnrm-system
   219  spec:
   220    ports:
   221    - name: controller-manager
   222      port: 443
   223    - name: metrics
   224      port: 8888
   225    selector:
   226      cnrm.cloud.google.com/component: cnrm-controller-manager
   227      cnrm.cloud.google.com/system: "true"
   228  `, `
   229  apiVersion: apps/v1
   230  kind: StatefulSet
   231  metadata:
   232    labels:
   233      cnrm.cloud.google.com/component: cnrm-controller-manager
   234      cnrm.cloud.google.com/system: "true"
   235    name: cnrm-controller-manager
   236    namespace: cnrm-system
   237  spec:
   238    selector:
   239      matchLabels:
   240        cnrm.cloud.google.com/component: cnrm-controller-manager
   241        cnrm.cloud.google.com/system: "true"
   242    serviceName: cnrm-manager
   243    template:
   244      metadata:
   245        labels:
   246          cnrm.cloud.google.com/component: cnrm-controller-manager
   247          cnrm.cloud.google.com/system: "true"
   248  `}
   249  
   250  var ClusterModeOnlyGCPComponents = []string{`
   251  apiVersion: v1
   252  kind: ServiceAccount
   253  metadata:
   254    name: cnrm-controller-manager
   255    namespace: cnrm-system
   256  `, `
   257  apiVersion: v1
   258  kind: Service
   259  metadata:
   260    name: cnrm-manager
   261    namespace: cnrm-system
   262  spec:
   263    ports:
   264    - name: controller-manager
   265      port: 443
   266    - name: metrics
   267      port: 8888
   268    selector:
   269      cnrm.cloud.google.com/component: cnrm-controller-manager
   270      cnrm.cloud.google.com/system: "true"
   271  `, `
   272  apiVersion: apps/v1
   273  kind: StatefulSet
   274  metadata:
   275    labels:
   276      cnrm.cloud.google.com/component: cnrm-controller-manager
   277      cnrm.cloud.google.com/system: "true"
   278    name: cnrm-controller-manager
   279    namespace: cnrm-system
   280  spec:
   281    selector:
   282      matchLabels:
   283        cnrm.cloud.google.com/component: cnrm-controller-manager
   284        cnrm.cloud.google.com/system: "true"
   285    serviceName: cnrm-manager
   286    template:
   287      metadata:
   288        labels:
   289          cnrm.cloud.google.com/component: cnrm-controller-manager
   290          cnrm.cloud.google.com/system: "true"
   291      spec:
   292        volumes:
   293        - name: gcp-service-account
   294          secret:
   295            secretName: gcp-key
   296  `}
   297  
   298  var PerNamespaceControllerManagerPod = `apiVersion: v1
   299  kind: Pod
   300  metadata:
   301    labels:
   302      cnrm.cloud.google.com/component: cnrm-controller-manager
   303      cnrm.cloud.google.com/system: "true"
   304    name: cnrm-controller-manager-12345-0
   305    namespace: cnrm-system
   306  spec:
   307    containers:
   308    - name: manager
   309      image: test-image
   310  `
   311  
   312  func GetSharedComponentsManifest() []string {
   313  	res := make([]string, 0)
   314  	res = append(res, FooCRD, SystemNs)
   315  	return res
   316  }
   317  
   318  func GetManifestsWithAlphaAndBetaCRDs() []string {
   319  	res := make([]string, 0)
   320  	res = append(res, FooCRD, barCRD, SystemNs)
   321  	return res
   322  }
   323  
   324  func GetManifestsWithAlphaCRD() []string {
   325  	res := make([]string, 0)
   326  	res = append(res, FooCRD, SystemNs)
   327  	return res
   328  }
   329  
   330  func GetManifestsWithBetaCRD() []string {
   331  	res := make([]string, 0)
   332  	res = append(res, barCRD, SystemNs)
   333  	return res
   334  }
   335  
   336  func GetManifestsWithNoCRD() []string {
   337  	res := make([]string, 0)
   338  	res = append(res, SystemNs)
   339  	return res
   340  }
   341  
   342  func GetManifestsWithNonKCCCRD() []string {
   343  	res := make([]string, 0)
   344  	res = append(res, nonKCCCRD, FooCRD, SystemNs)
   345  	return res
   346  }
   347  
   348  func GetManifestsWithDefectiveCRD() []string {
   349  	res := make([]string, 0)
   350  	res = append(res, defectiveCRD, SystemNs)
   351  	return res
   352  }
   353  
   354  func GetClusterModeGCPManifest() []string {
   355  	res := make([]string, 0)
   356  	res = append(res, GetSharedComponentsManifest()...)
   357  	res = append(res, ClusterModeOnlyGCPComponents...)
   358  	return res
   359  }
   360  
   361  func GetClusterModeWorkloadIdentityManifest() []string {
   362  	res := make([]string, 0)
   363  	res = append(res, GetSharedComponentsManifest()...)
   364  	res = append(res, ClusterModeOnlyWorkloadIdentityComponents...)
   365  	return res
   366  }
   367  
   368  func GetPerNamespaceManifest() []string {
   369  	res := make([]string, 0)
   370  	res = append(res, NamespacedComponentsTemplate...)
   371  	return res
   372  }
   373  
   374  func ManuallyReplaceGSA(components []string, saName string) []string {
   375  	res := make([]string, 0)
   376  	for _, s := range components {
   377  		s = strings.ReplaceAll(s, "${SERVICE_ACCOUNT?}", saName)
   378  		res = append(res, s)
   379  	}
   380  	return res
   381  }
   382  
   383  func ManuallyReplaceSecretVolume(components []string, secretName string) []string {
   384  	res := make([]string, 0)
   385  	for _, s := range components {
   386  		s = strings.ReplaceAll(s, "gcp-key", secretName)
   387  		res = append(res, s)
   388  	}
   389  	return res
   390  }
   391  
   392  func ManuallyModifyNamespaceTemplates(t *testing.T, template []string, nsName, saName string, userProjectOverride bool, billingProject string, c client.Client) []string {
   393  	var res []string
   394  	nsId, err := cluster.GetNamespaceID(k8s.OperatorNamespaceIDConfigMapNN, c, context.TODO(), nsName)
   395  	if err != nil {
   396  		t.Fatalf("error getting the id for namespace %v", err)
   397  	}
   398  	for _, s := range template {
   399  		applied := s
   400  		if strings.Contains(s, "kind: StatefulSet") {
   401  			if billingProject != "" {
   402  				applied = strings.ReplaceAll(applied,
   403  					`args: ["--scoped-namespace=${NAMESPACE?}", "--stderrthreshold=INFO", "--prometheus-scrape-endpoint=:8888"`,
   404  					`args: ["--scoped-namespace=${NAMESPACE?}", "--stderrthreshold=INFO", "--prometheus-scrape-endpoint=:8888", "--billing-project=`+billingProject+`"`,
   405  				)
   406  			}
   407  
   408  			if userProjectOverride {
   409  				applied = strings.ReplaceAll(applied,
   410  					`args: ["--scoped-namespace=${NAMESPACE?}", "--stderrthreshold=INFO", "--prometheus-scrape-endpoint=:8888"`,
   411  					`args: ["--scoped-namespace=${NAMESPACE?}", "--stderrthreshold=INFO", "--prometheus-scrape-endpoint=:8888", "--user-project-override=true"`,
   412  				)
   413  			}
   414  
   415  			applied = strings.ReplaceAll(applied, "cnrm-controller-manager-${NAMESPACE?}", "cnrm-controller-manager-"+nsId)
   416  			applied = strings.ReplaceAll(applied, k8s.NamespacedManagerServiceTmpl, k8s.NamespacedManagerServicePrefix+nsId)
   417  		}
   418  		if strings.Contains(s, "name: cnrm-manager-${NAMESPACE?}") {
   419  			applied = strings.ReplaceAll(applied, k8s.NamespacedManagerServiceTmpl, k8s.NamespacedManagerServicePrefix+nsId)
   420  		}
   421  		applied = strings.ReplaceAll(applied, "${SERVICE_ACCOUNT?}", saName)
   422  		applied = strings.ReplaceAll(applied, "${NAMESPACE?}", nsName)
   423  		u := ToUnstructured(t, applied)
   424  		labels := u.GetLabels()
   425  		labels[k8s.ConfigConnectorContextNamespaceLabel] = nsName
   426  		u.SetLabels(labels)
   427  		applied = ToString(t, u)
   428  		res = append(res, applied)
   429  	}
   430  	return res
   431  }
   432  
   433  func ToUnstructured(t *testing.T, objStr string) *unstructured.Unstructured {
   434  	u := &unstructured.Unstructured{}
   435  	b := []byte(objStr)
   436  	err := yaml.Unmarshal(b, u)
   437  	if err != nil {
   438  		t.Fatalf("error unmarshalling bytes to unstruct: %v", err)
   439  	}
   440  	return u
   441  }
   442  
   443  func ToString(t *testing.T, u *unstructured.Unstructured) string {
   444  	json, err := u.MarshalJSON()
   445  	if err != nil {
   446  		t.Fatalf("error marshalling unstructured to json: %v", err)
   447  	}
   448  	y, err := yaml.JSONToYAML(json)
   449  	if err != nil {
   450  		t.Fatalf("error converting json to yaml: %v", err)
   451  	}
   452  	return string(y)
   453  }
   454  
   455  func ParseObjects(t *testing.T, ctx context.Context, objects []string) *manifest.Objects {
   456  	objs := strings.Join(objects, "---\n")
   457  	m, err := manifest.ParseObjects(ctx, objs)
   458  	if err != nil {
   459  		t.Fatalf("while parsing objects: %v", err)
   460  	}
   461  	return m
   462  }
   463  

View as plain text