namespaced.go

Documentation: github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/controllers/configconnectorcontext

     1  // Copyright 2022 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package configconnectorcontext
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"strings"
    21  	"time"
    22  
    23  	corev1beta1 "github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/apis/core/v1beta1"
    24  	"github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/controllers"
    25  	"github.com/GoogleCloudPlatform/k8s-config-connector/operator/pkg/k8s"
    26  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/cluster"
    27  
    28  	"github.com/pkg/errors"
    29  	appsv1 "k8s.io/api/apps/v1"
    30  	corev1 "k8s.io/api/core/v1"
    31  	rbacv1 "k8s.io/api/rbac/v1"
    32  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    33  	"k8s.io/apimachinery/pkg/util/wait"
    34  	"sigs.k8s.io/controller-runtime/pkg/client"
    35  	"sigs.k8s.io/kubebuilder-declarative-pattern/pkg/patterns/declarative/pkg/manifest"
    36  )
    37  
    38  func removeNamespacedComponents(ctx context.Context, c client.Client, objects []*manifest.Object) error {
    39  	for _, obj := range objects {
    40  		if err := controllers.DeleteObject(ctx, c, obj.UnstructuredObject()); err != nil {
    41  			return err
    42  		}
    43  	}
    44  	return nil
    45  }
    46  
    47  func transformNamespacedComponentTemplates(ctx context.Context, c client.Client, ccc *corev1beta1.ConfigConnectorContext, namespacedTemplates []*manifest.Object) ([]*manifest.Object, error) {
    48  	transformedObjs := make([]*manifest.Object, 0, len(namespacedTemplates))
    49  	for _, obj := range namespacedTemplates {
    50  		processed := obj
    51  		if controllers.IsControllerManagerService(processed) {
    52  			var err error
    53  			processed, err = handleControllerManagerService(ctx, c, ccc, processed)
    54  			if err != nil {
    55  				return nil, err
    56  			}
    57  		}
    58  		if controllers.IsControllerManagerStatefulSet(processed) {
    59  			var err error
    60  			processed, err = handleControllerManagerStatefulSet(ctx, c, ccc, processed)
    61  			if err != nil {
    62  				return nil, err
    63  			}
    64  		}
    65  		processed, err := replaceNamespacePattern(processed, ccc.Namespace)
    66  		if err != nil {
    67  			return nil, err
    68  		}
    69  		if processed.Kind == rbacv1.ServiceAccountKind && strings.HasPrefix(processed.GetName(), k8s.ServiceAccountNamePrefix) {
    70  			processed, err = controllers.AnnotateServiceAccountObject(processed, ccc.Spec.GoogleServiceAccount)
    71  			if err != nil {
    72  				return nil, errors.Wrap(err, fmt.Sprintf("error annotating ServiceAccount %v/%v", obj.UnstructuredObject().GetNamespace(), obj.UnstructuredObject().GetName()))
    73  			}
    74  		}
    75  		transformedObjs = append(transformedObjs, processed)
    76  	}
    77  	return transformedObjs, nil
    78  }
    79  
    80  func handleControllerManagerService(ctx context.Context, c client.Client, ccc *corev1beta1.ConfigConnectorContext, obj *manifest.Object) (*manifest.Object, error) {
    81  	u := obj.UnstructuredObject().DeepCopy()
    82  	nsId, err := cluster.GetNamespaceID(k8s.OperatorNamespaceIDConfigMapNN, c, ctx, ccc.Namespace)
    83  	if err != nil {
    84  		return nil, fmt.Errorf("error getting namespace id for namespace %v: %v", ccc.Namespace, err)
    85  	}
    86  	u.SetName(strings.ReplaceAll(u.GetName(), "${NAMESPACE?}", nsId))
    87  	if err := removeStaleControllerManagerService(ctx, c, ccc.Namespace, u.GetName()); err != nil {
    88  		return nil, fmt.Errorf("error deleting stale Services for watched namespace %v: %v", ccc.Namespace, err)
    89  	}
    90  	return manifest.NewObject(u)
    91  }
    92  
    93  func removeStaleControllerManagerService(ctx context.Context, c client.Client, ns string, validSts string) error {
    94  	// List existing controller-manager services for the given namespace, delete stale ones if any
    95  	// stale services could come from legacy naming or namespaceId changes.
    96  	svcList := &corev1.ServiceList{}
    97  	if err := c.List(ctx, svcList, client.InNamespace(k8s.CNRMSystemNamespace),
    98  		client.MatchingLabels{k8s.NamespacedComponentLabel: ns}); err != nil {
    99  		return fmt.Errorf("error listing existing %v Services for watched namespace %v: %v", k8s.KCCControllerManagerComponent, ns, err)
   100  	}
   101  	for _, svc := range svcList.Items {
   102  		if strings.HasPrefix(svc.Name, k8s.NamespacedManagerServicePrefix) && svc.Name != validSts {
   103  			if err := controllers.DeleteObject(ctx, c, &svc); err != nil {
   104  				return err
   105  			}
   106  		}
   107  	}
   108  	return nil
   109  }
   110  
   111  func handleControllerManagerStatefulSet(ctx context.Context, c client.Client, ccc *corev1beta1.ConfigConnectorContext, obj *manifest.Object) (*manifest.Object, error) {
   112  	u := obj.UnstructuredObject().DeepCopy()
   113  
   114  	nsId, err := cluster.GetNamespaceID(k8s.OperatorNamespaceIDConfigMapNN, c, ctx, ccc.Namespace)
   115  	if err != nil {
   116  		return nil, fmt.Errorf("error getting namespace id for namespace %v: %v", ccc.Namespace, err)
   117  	}
   118  
   119  	u.SetName(strings.ReplaceAll(u.GetName(), "${NAMESPACE?}", nsId))
   120  
   121  	serviceName, found, err := unstructured.NestedString(u.Object, "spec", "serviceName")
   122  	if err != nil || !found {
   123  		return nil, fmt.Errorf("couldn't resolve serviceName in StatefulSet %v for watched namespace %v: %v", u.GetName(), ccc.Namespace, err)
   124  	}
   125  	if err := unstructured.SetNestedField(u.Object, strings.ReplaceAll(serviceName, "${NAMESPACE?}", nsId), "spec", "serviceName"); err != nil {
   126  		return nil, err
   127  	}
   128  
   129  	if ccc.GetRequestProjectPolicy() == k8s.ResourceProjectPolicy {
   130  		if err := enableUserProjectOverride(u); err != nil {
   131  			return nil, fmt.Errorf("error enabling %v in StatefulSet %v for watched namespace %v: %v", k8s.UserProjectOverrideFlag, u.GetName(), ccc.Namespace, err)
   132  		}
   133  	}
   134  
   135  	if ccc.GetRequestProjectPolicy() == k8s.BillingProjectPolicy {
   136  		if err := enableUserProjectOverride(u); err != nil {
   137  			return nil, fmt.Errorf("error enabling %v in StatefulSet %v for watched namespace %v: %v", k8s.UserProjectOverrideFlag, u.GetName(), ccc.Namespace, err)
   138  		}
   139  		if err := enableBillingProject(u, ccc.Spec.BillingProject); err != nil {
   140  			return nil, fmt.Errorf("error enabling %v in StatefulSet %v for watched namespace %v: %v", k8s.BillingProjectFlag, u.GetName(), ccc.Namespace, err)
   141  		}
   142  	}
   143  
   144  	if err := removeStaleControllerManagerStatefulSet(ctx, c, ccc.Namespace, u.GetName()); err != nil {
   145  		return nil, fmt.Errorf("error deleting stale StatefulSet for watched namespace %v: %v", ccc.Namespace, err)
   146  	}
   147  
   148  	return manifest.NewObject(u)
   149  }
   150  
   151  func enableUserProjectOverride(u *unstructured.Unstructured) error {
   152  	return setFlagForManagerContainer(u, k8s.UserProjectOverrideFlag, "true")
   153  }
   154  
   155  func enableBillingProject(u *unstructured.Unstructured, flagValue string) error {
   156  	return setFlagForManagerContainer(u, k8s.BillingProjectFlag, flagValue)
   157  }
   158  
   159  func findManagerContainer(containers []interface{}) (managerContainer map[string]interface{}, index int, err error) {
   160  	for i, container := range containers {
   161  		containerAsMap, ok := container.(map[string]interface{})
   162  		if !ok {
   163  			return nil, 0, fmt.Errorf("couldn't convert container configuration %v to a map", container)
   164  		}
   165  		name, found, err := unstructured.NestedString(containerAsMap, "name")
   166  		if err != nil || !found {
   167  			return nil, 0, fmt.Errorf("couldn't resolve name of container configuration %v: %v", container, err)
   168  		}
   169  		if name == k8s.CNRMManagerContainerName {
   170  			return containerAsMap, i, nil
   171  		}
   172  	}
   173  	return nil, 0, fmt.Errorf("no manager container found")
   174  }
   175  
   176  // A helper method to add optional flags for manager container.
   177  func setFlagForManagerContainer(u *unstructured.Unstructured, flag string, flagValue string) error {
   178  	containersPath := []string{"spec", "template", "spec", "containers"} // Path to container configurations in a StatefulSet
   179  	containers, found, err := unstructured.NestedSlice(u.Object, containersPath...)
   180  	if err != nil || !found {
   181  		return fmt.Errorf("couldn't resolve containers: %w", err)
   182  	}
   183  
   184  	managerContainer, index, err := findManagerContainer(containers)
   185  	if err != nil {
   186  		return fmt.Errorf("error finding manager container: %v", err)
   187  	}
   188  	args, found, err := unstructured.NestedStringSlice(managerContainer, "args")
   189  	if err != nil {
   190  		return fmt.Errorf("couldn't resolve args of manager container %v: %w", managerContainer, err)
   191  	}
   192  	if !found {
   193  		args = make([]string, 0)
   194  	}
   195  	newArgs := removeFlagFromArgs(args, flag)
   196  	newArgs = append(newArgs, flag+"="+flagValue)
   197  	if err := unstructured.SetNestedStringSlice(managerContainer, newArgs, "args"); err != nil {
   198  		return fmt.Errorf("error setting args in manager container: %v", err)
   199  	}
   200  
   201  	containers[index] = managerContainer
   202  	if err := unstructured.SetNestedSlice(u.Object, containers, containersPath...); err != nil {
   203  		return fmt.Errorf("error setting containers: %v", err)
   204  	}
   205  	return nil
   206  }
   207  
   208  func removeFlagFromArgs(args []string, flag string) []string {
   209  	newArgs := make([]string, 0)
   210  	for _, a := range args {
   211  		if !strings.HasPrefix(a, flag) {
   212  			newArgs = append(newArgs, a)
   213  		}
   214  	}
   215  	return newArgs
   216  }
   217  
   218  func removeStaleControllerManagerStatefulSet(ctx context.Context, c client.Client, ns string, validSts string) error {
   219  	// List existing controller-manager statefulsets for the given namespace, delete stale ones if any
   220  	// stale statefulsets could come from legacy naming or namespaceId changes.
   221  	stsList := &appsv1.StatefulSetList{}
   222  	if err := c.List(ctx, stsList, client.InNamespace(k8s.CNRMSystemNamespace),
   223  		client.MatchingLabels{k8s.KCCSystemComponentLabel: k8s.KCCControllerManagerComponent, k8s.NamespacedComponentLabel: ns}); err != nil {
   224  		return fmt.Errorf("error listing existing %v StatefulSets for watched namespace %v: %v", k8s.KCCControllerManagerComponent, ns, err)
   225  	}
   226  
   227  	hasStale := false
   228  	for _, sts := range stsList.Items {
   229  		if sts.Name != validSts {
   230  			hasStale = true
   231  			if err := controllers.DeleteObject(ctx, c, &sts); err != nil {
   232  				return err
   233  			}
   234  		}
   235  	}
   236  
   237  	if hasStale {
   238  		b := wait.Backoff{
   239  			Duration: time.Second,
   240  			Factor:   1.2,
   241  			Steps:    12,
   242  		}
   243  		podList := &corev1.PodList{}
   244  		if err := wait.ExponentialBackoff(b, func() (done bool, err error) {
   245  			if err := c.List(ctx, podList, client.InNamespace(k8s.CNRMSystemNamespace),
   246  				client.MatchingLabels{k8s.KCCSystemComponentLabel: k8s.KCCControllerManagerComponent, k8s.NamespacedComponentLabel: ns}); err != nil {
   247  				return false, errors.Wrap(err, "error listing controller pods")
   248  			}
   249  			if len(podList.Items) == 0 {
   250  				return true, nil
   251  			}
   252  			if len(podList.Items) == 1 {
   253  				pod := &podList.Items[0]
   254  				for _, owner := range pod.OwnerReferences {
   255  					if owner.Kind == "StatefulSet" && owner.Name == validSts {
   256  						return true, nil
   257  					}
   258  				}
   259  			}
   260  			return false, nil
   261  		}); err != nil {
   262  			return errors.Wrap(err, "error waiting for stale controller pods to be deleted")
   263  		}
   264  	}
   265  	return nil
   266  }
   267  
   268  func replaceNamespacePattern(obj *manifest.Object, ns string) (*manifest.Object, error) {
   269  	bytes, err := obj.JSON()
   270  	if err != nil {
   271  		return nil, errors.Wrap(err, fmt.Sprintf("error marshalling object %v", obj.UnstructuredObject()))
   272  	}
   273  	str := string(bytes)
   274  	str = strings.ReplaceAll(str, "${NAMESPACE?}", ns)
   275  	newObj, err := manifest.ParseJSONToObject([]byte(str))
   276  	if err != nil {
   277  		return nil, errors.Wrap(err, fmt.Sprintf("error unmarshalling object %v", obj.UnstructuredObject()))
   278  	}
   279  	return newObj, nil
   280  }
   281
View as plain text