managedfields.go

Documentation: github.com/GoogleCloudPlatform/k8s-config-connector/pkg/k8s

     1  // Copyright 2022 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package k8s
    16  
    17  import (
    18  	"bytes"
    19  	"fmt"
    20  
    21  	corekccv1alpha1 "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/apis/core/v1alpha1"
    22  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/deepcopy"
    23  	"github.com/GoogleCloudPlatform/k8s-config-connector/pkg/util"
    24  
    25  	apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    26  	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    27  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    28  	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
    29  	"sigs.k8s.io/structured-merge-diff/v4/schema"
    30  	"sigs.k8s.io/structured-merge-diff/v4/typed"
    31  	"sigs.k8s.io/structured-merge-diff/v4/value"
    32  )
    33  
    34  /*
    35  
    36  This file contains all the functionality around field management in KCC.
    37  KCC supports the concept of "externally-managed" fields
    38  (https://cloud.google.com/config-connector/docs/concepts/managing-fields-externally),
    39  which respects the live value for fields on the underlying API that no Kubernetes user manages.
    40  Thus, we need to parse and perform operations using the core Kubernetes field management metadata.
    41  
    42  For more information on core Kubernetes field management, see the Server-Side
    43  Apply Kubernetes documentation:
    44  https://kubernetes.io/docs/reference/using-api/server-side-apply
    45  
    46  This package makes heavy use of the functionality exposed by the
    47  `sigs.k8s.io/structured-merge-diff` library. This is the library used by the
    48  Kubernetes API server to perform all its operations surrounding its managed
    49  field metadata.
    50  
    51  The following packages from `sigs.k8s.io/structured-merge-diff` are leveraged:
    52  
    53  * sigs.k8s.io/structured-merge-diff/v4/fieldpath: We make use of the
    54    `fieldpath.Set` type. This exposes standard set operations like Union,
    55    Difference, Intersection for sets of fields. A JSON-encoding of this type is
    56    what is directly stored in the core Kubernetes
    57    `metadata.managedFields[].fieldsV1` field.
    58  
    59  * sigs.k8s.io/structured-merge-diff/v4/value: We make use of the `value.Value`
    60    type. This is just a wrapper type for a JSON type. All we need to do with
    61    this structure is create one with a map[string]interface{}, and extract the
    62    updated map[string]interface{} at the end.
    63  
    64  * sigs.k8s.io/structured-merge-diff/v4/typed: We make use of the
    65    `typed.TypedValue` type. The struct itself is just a wrapper for a pair of a
    66    `value.Value` and a `schema.Schema` (explained below). However, it exposes a
    67    bunch of handy functions around comparing and extracting values from objects.
    68    For example:
    69    * The `Compare` function takes two objects (and old state and an updated
    70      state) and returns a Comparison object with field sets of everything that
    71      was added, modified, and removed.
    72    * The `FieldSet` function returns a set of all the fields that show up in the
    73      object. This is useful for comparing to the core Kubernetes managed field
    74      metadata.
    75    * The `Merge` function takes a partial state and merges it with an object.
    76    * The `ExtractItems` function can take a field set and extract a partial
    77      state from an object with just those fields represented.
    78    * The `RemoveItems` function can take a field set and remove all those fields
    79      from an object.
    80  
    81  * sigs.k8s.io/structured-merge-diff/v4/schema: We use the `schema.Schema` type.
    82    This is a schema type exclusive to structured-merge-diff (abbreviated as SMD)
    83    operations. It is a requirement to create a `typed.TypedValue`, but no
    84    operations are used on it directly. This just decouples the schema
    85    information relevant to SMD operations from other schema types like
    86    `apiextensions.JSONSchemaProps`.
    87  
    88  */
    89  
    90  func IsK8sManaged(key string, specObj map[string]interface{}, managedFields *fieldpath.Set) bool {
    91  	pe := fieldpath.PathElement{FieldName: &key}
    92  	if managedFields == nil {
    93  		// If no managed field information present, treat values specified in the
    94  		// spec as k8s-managed.
    95  		_, ok := specObj[key]
    96  		return ok
    97  	}
    98  	if managedFields.Members.Has(pe) {
    99  		return true
   100  	}
   101  	// We must also check the nested objects within the managed fields for management
   102  	// in order to handle the following cases:
   103  	// - Sensitive fields are converted from strings to complex reference objects
   104  	// - Maps, though nested objects and technically able to be merged, must be treated
   105  	//   as atomic in order to allow for users to clear entries
   106  	_, found := managedFields.Children.Get(pe)
   107  	return found
   108  }
   109  
   110  // ConstructManagedFieldsV1Set takes the given managed field entries and constructs a
   111  // set of all the k8s-managed fields from the spec.
   112  func ConstructManagedFieldsV1Set(managedFields []v1.ManagedFieldsEntry) (*fieldpath.Set, error) {
   113  	res := fieldpath.NewSet()
   114  	for _, managedFieldEntry := range managedFields {
   115  		if managedFieldEntry.Manager == ControllerManagedFieldManager {
   116  			continue
   117  		}
   118  		if managedFieldEntry.FieldsType != ManagedFieldsTypeFieldsV1 {
   119  			return nil, fmt.Errorf(
   120  				"expected managed field entry for manager '%v' and operation '%v' of type '%v', got type '%v'",
   121  				managedFieldEntry.Manager, managedFieldEntry.Operation, ManagedFieldsTypeFieldsV1,
   122  				managedFieldEntry.FieldsType)
   123  		}
   124  		fieldsV1 := managedFieldEntry.FieldsV1
   125  		if managedFieldEntry.FieldsV1 == nil {
   126  			return nil, fmt.Errorf("managed field entry for manager '%v' and operation '%v' has empty fieldsV1",
   127  				managedFieldEntry.Manager, managedFieldEntry.Operation)
   128  		}
   129  		entrySet := fieldpath.NewSet()
   130  		if err := entrySet.FromJSON(bytes.NewReader(fieldsV1.Raw)); err != nil {
   131  			return nil, fmt.Errorf("error marshaling managed fields for manager '%v' and operation '%v' from JSON: %w",
   132  				managedFieldEntry.Manager, managedFieldEntry.Operation, err)
   133  		}
   134  		specFieldName := "spec"
   135  		specSet, found := entrySet.Children.Get(fieldpath.PathElement{FieldName: &specFieldName})
   136  		if !found {
   137  			continue
   138  		}
   139  		res = res.Union(specSet)
   140  	}
   141  	return res, nil
   142  }
   143  
   144  func containsUnsupportedFieldTypes(managedFields []v1.ManagedFieldsEntry) bool {
   145  	for _, entry := range managedFields {
   146  		// Only FieldsV1 is currently supported.
   147  		if entry.FieldsType != ManagedFieldsTypeFieldsV1 {
   148  			return true
   149  		}
   150  	}
   151  	return false
   152  }
   153  
   154  func GetK8sManagedFields(u *unstructured.Unstructured) (*fieldpath.Set, error) {
   155  	managedFields := u.GetManagedFields()
   156  	if managedFields != nil && !containsUnsupportedFieldTypes(managedFields) {
   157  		res, err := ConstructManagedFieldsV1Set(managedFields)
   158  		if err != nil {
   159  			return nil, err
   160  		}
   161  		return res, nil
   162  	}
   163  	return nil, nil
   164  }
   165  
   166  // OverlayManagedFieldsOntoState overlays the fields managed by Kubernetes managers onto the
   167  // KRM-ified live state.
   168  //
   169  // The return value is the union of stateAsKRM with managed fields from spec.
   170  func OverlayManagedFieldsOntoState(spec, stateAsKRM map[string]interface{}, managedFields *fieldpath.Set,
   171  	jsonSchema *apiextensions.JSONSchemaProps, hierarchicalRefs []corekccv1alpha1.HierarchicalReference) (map[string]interface{}, error) {
   172  	config, err := overlayManagedFieldsOntoState(spec, stateAsKRM, managedFields, jsonSchema)
   173  	if err != nil {
   174  		return nil, err
   175  	}
   176  
   177  	// Add the resource's hierarchical reference back to the config if the
   178  	// resource supports hierarchical references. We do this because
   179  	// overlayManagedFieldsOntoState() strips out the hierarchical reference if
   180  	// it was defaulted by the KCC webhook/controller. This is because the
   181  	// resource's managedFields metadata doesn't include the hierarchcial
   182  	// reference if it was defaulted by KCC rather than explicitly set by the
   183  	// user. Since we know that a resource's hierarchical reference is part of
   184  	// the desired state even if it's not in the managedFields metadata, let's
   185  	// add it back in.
   186  	// TODO(b/184319410): Come up with a more generic solution to ensure that
   187  	// fields added by webhooks/controllers are preserved as part of the
   188  	// desired state if they truly belong there.
   189  	if len(hierarchicalRefs) > 0 {
   190  		config, err = addHierarchicalReferenceToConfig(config, spec, hierarchicalRefs)
   191  		if err != nil {
   192  			return nil, fmt.Errorf("error adding hierarchical reference to config: %v", err)
   193  		}
   194  	}
   195  
   196  	return config, nil
   197  }
   198  
   199  func overlayManagedFieldsOntoState(spec, stateAsKRM map[string]interface{}, managedFields *fieldpath.Set,
   200  	jsonSchema *apiextensions.JSONSchemaProps) (map[string]interface{}, error) {
   201  	if jsonSchema == nil {
   202  		return nil, fmt.Errorf("JSON schema is required")
   203  	}
   204  	specSchema, ok := jsonSchema.Properties["spec"]
   205  	if !ok {
   206  		if spec != nil && len(spec) > 0 {
   207  			return nil, fmt.Errorf("cannot parse spec with no spec schema available")
   208  		}
   209  		return make(map[string]interface{}), nil
   210  	}
   211  
   212  	// Wrap the spec and live state objects with `typed.TypedValue`
   213  	// objects, as this type has methods defined on it for operations
   214  	// like calculating field sets, extracting partial states, and
   215  	// merging partial states.
   216  	smdSchema := jsonSchemaToSMDSchema(&specSchema)
   217  	specAsTyped, err := toTypedValue(spec, smdSchema)
   218  	if err != nil {
   219  		return nil, fmt.Errorf("error converting spec to typed value: %w", err)
   220  	}
   221  	stateAsTyped, err := toTypedValue(stateAsKRM, smdSchema)
   222  	if err != nil {
   223  		return nil, fmt.Errorf("error converting state to typed value: %w", err)
   224  	}
   225  
   226  	if managedFields == nil {
   227  		managedFields = &fieldpath.Set{}
   228  	}
   229  
   230  	// Construct a set of only the leaves in order to avoid the behavior
   231  	// during ValueType.Merge where a parent element being a member of the
   232  	// set (i.e. listed as "." in the managed fields) causes the child
   233  	// elements to be left unmerged and the parent object taken wholesale.
   234  	managedFields = managedFields.Leaves()
   235  
   236  	// Treat _atomic_ list fields as k8s-managed. We do this since the controller
   237  	// assumes management over atomic lists to be able to default values inside
   238  	// objects in lists. Therefore, to avoid suppressing intentional diffs,
   239  	// treat any atomic list as k8s-managed.
   240  	atomicListFields, err := getAtomicListFields(smdSchema)
   241  	if err != nil {
   242  		return nil, fmt.Errorf("error getting atomic list fields: %w", err)
   243  	}
   244  	specFieldSet, err := specAsTyped.ToFieldSet()
   245  	if err != nil {
   246  		return nil, fmt.Errorf("error constructing field set for spec: %w", err)
   247  	}
   248  	atomicListFields = atomicListFields.Intersection(specFieldSet)
   249  	managedFields = managedFields.Union(atomicListFields)
   250  
   251  	// Extract k8s-managed fields from the spec, and then merge them with the live
   252  	// state. In cases where the k8s-managed state and the live state both have the
   253  	// same field, the value from the k8s-managed state is taken.
   254  	k8sManagedPartialState := specAsTyped.ExtractItems(managedFields)
   255  	overlaidState, err := stateAsTyped.Merge(k8sManagedPartialState)
   256  	if err != nil {
   257  		return nil, fmt.Errorf("error merging partial managed state with live state: %w", err)
   258  	}
   259  
   260  	// Unwrap the `typed.TypedValue` back to a map[string]interface{} that
   261  	// the rest of reconciliation expects.
   262  	overlaidStateRaw := overlaidState.AsValue().Unstructured()
   263  	if overlaidStateRaw == nil {
   264  		return make(map[string]interface{}), nil
   265  	}
   266  	res, ok := overlaidStateRaw.(map[string]interface{})
   267  	if !ok {
   268  		return nil, fmt.Errorf("overlaid state unstructured not of type map[string]interface{}")
   269  	}
   270  	return res, nil
   271  }
   272  
   273  // Construct the trimmed spec that only contains k8s managed fields.
   274  //
   275  // The DCL SDK's Apply() function can take a partial state that only contains fields that users have
   276  // an opinion on. Here we will look into the managed-fields set and trim the full spec to only preserve
   277  // fields that are k8s-managed (i.e. users want KCC to enforce those fields to their desired state).
   278  // DCL will take the generated partial state, enforce specified fields and ignore unspecified fields
   279  // by preserving live values from the underlying API.
   280  func ConstructTrimmedSpecWithManagedFields(resource *Resource, jsonSchema *apiextensions.JSONSchemaProps,
   281  	hierarchicalRefs []corekccv1alpha1.HierarchicalReference) (map[string]interface{}, error) {
   282  	if resource.ManagedFields == nil {
   283  		// If no managed field information present, treat values specified in the
   284  		// spec as k8s-managed.
   285  		return resource.Spec, nil
   286  	}
   287  	if resource.Spec == nil {
   288  		return nil, nil
   289  	}
   290  	// construct an empty state map to overlay onto
   291  	emptyState := make(map[string]interface{})
   292  	trimmed, err := OverlayManagedFieldsOntoState(resource.Spec, emptyState,
   293  		resource.ManagedFields, jsonSchema, hierarchicalRefs)
   294  	if err != nil {
   295  		return nil, fmt.Errorf("error constructing trimmed state with managed fields: %w", err)
   296  	}
   297  	return trimmed, nil
   298  }
   299  
   300  // getAtomicListFields returns a field set of all atomic list fields in the
   301  // given schema.
   302  func getAtomicListFields(s *schema.Schema) (*fieldpath.Set, error) {
   303  	if len(s.Types) != 1 {
   304  		return nil, fmt.Errorf("expected schema to have 1 type, got %v", len(s.Types))
   305  	}
   306  	typeDef := s.Types[0]
   307  	if typeDef.Map == nil {
   308  		return nil, fmt.Errorf("type definition is not map")
   309  	}
   310  	return getAtomicListFieldsFromMap(typeDef.Map), nil
   311  }
   312  
   313  func getAtomicListFieldsFromMap(m *schema.Map) *fieldpath.Set {
   314  	res := &fieldpath.Set{}
   315  	for _, structField := range m.Fields {
   316  		fieldName := structField.Name
   317  		pe := fieldpath.PathElement{FieldName: &fieldName}
   318  		fieldAtom := structField.Type.Inlined
   319  		if fieldAtom.List != nil {
   320  			switch fieldAtom.List.ElementRelationship {
   321  			case schema.Atomic:
   322  				res.Members.Insert(pe)
   323  			case schema.Associative:
   324  				elemAtom := fieldAtom.List.ElementType.Inlined
   325  				if elemAtom.Map != nil {
   326  					nestedSet := getAtomicListFieldsFromMap(elemAtom.Map)
   327  					if !nestedSet.Empty() {
   328  						insertChild(res, nestedSet, pe)
   329  					}
   330  				}
   331  			}
   332  		} else if fieldAtom.Map != nil {
   333  			nestedSet := getAtomicListFieldsFromMap(fieldAtom.Map)
   334  			if !nestedSet.Empty() {
   335  				insertChild(res, nestedSet, pe)
   336  			}
   337  		}
   338  	}
   339  	return res
   340  }
   341  
   342  func insertChild(set, childSet *fieldpath.Set, pe fieldpath.PathElement) {
   343  	childSetPtr := set.Children.Descend(pe)
   344  	*childSetPtr = *childSet
   345  }
   346  
   347  func toTypedValue(obj map[string]interface{}, smdSchema *schema.Schema) (*typed.TypedValue, error) {
   348  	// The SMD schema constructed by jsonSchemaToSMDSchema is hardcoded to have
   349  	// only one type definition, with a blank name. Thus, our type reference
   350  	// here should refer to the empty string.
   351  	name := ""
   352  	return typed.AsTyped(value.NewValueInterface(obj),
   353  		smdSchema,
   354  		schema.TypeRef{
   355  			NamedType: &name,
   356  		},
   357  	)
   358  }
   359  
   360  // jsonSchemaToSMDSchema constructs a structured-merge-diff (SMD) schema.Schema
   361  // object from the given JSON schema. The SMD schema is a requirement to
   362  // create the typed.TypedValue objects.
   363  //
   364  // Note that `k8s.io/apiserver` defines a `TypeConverter` interface type that
   365  // is intended to directly generate a `typed.TypedValue` object from a
   366  // `runtime.Object`. However, the only concrete implementation takes a
   367  // schema as input in the form of `k8s.io/openapi/pkg/util/proto.Models`
   368  // rather than the `apiextensions.JSONSchemaProps` our controller has access
   369  // to. As the SMD schema generation is quite straightforward and allows for
   370  // only depending on `sigs.k8s.io/structured-merge-diff`, we choose to own
   371  // our own conversion logic.
   372  func jsonSchemaToSMDSchema(jsonSchema *apiextensions.JSONSchemaProps) *schema.Schema {
   373  	return &schema.Schema{
   374  		Types: []schema.TypeDef{
   375  			{
   376  				// Type definitions are named. However, since this schema only
   377  				// contains one definition, we can just leave the name blank.
   378  				Name: "",
   379  				Atom: jsonSchemaToAtom(jsonSchema),
   380  			},
   381  		}}
   382  }
   383  
   384  func jsonSchemaToAtom(jsonSchema *apiextensions.JSONSchemaProps) schema.Atom {
   385  	res := schema.Atom{}
   386  	var (
   387  		// scalarPtr is a helper function that allows us to easily reference
   388  		// the built-in schema.Scalar constants in a context where a pointer
   389  		// is required (since Go does not allow pointers to constants).
   390  		scalarPtr = func(s schema.Scalar) *schema.Scalar { return &s }
   391  
   392  		// scalarUnknown is a custom scalar type for use when the JSON schema
   393  		// has no schema available for map elements. We must include some
   394  		// sort of schema value, as map validation fails otherwise. Merges
   395  		// on custom scalar types are supported by the SMD library.
   396  		scalarUnknown = schema.Scalar("unknown")
   397  	)
   398  	switch jsonSchema.Type {
   399  	case "object":
   400  		res.Map = &schema.Map{}
   401  		if jsonSchema.Properties != nil {
   402  			for field, fieldSchema := range jsonSchema.Properties {
   403  				res.Map.Fields = append(res.Map.Fields, schema.StructField{
   404  					Name: field,
   405  					Type: schema.TypeRef{
   406  						Inlined: jsonSchemaToAtom(&fieldSchema),
   407  					},
   408  				})
   409  			}
   410  			break
   411  		}
   412  		if jsonSchema.AdditionalProperties != nil {
   413  			res.Map.ElementType = schema.TypeRef{
   414  				Inlined: jsonSchemaToAtom(jsonSchema.AdditionalProperties.Schema),
   415  			}
   416  			break
   417  		}
   418  		res.Map.ElementType = schema.TypeRef{
   419  			Inlined: schema.Atom{Scalar: &scalarUnknown},
   420  		}
   421  	case "array":
   422  		res.List = &schema.List{
   423  			ElementType: schema.TypeRef{
   424  				Inlined: jsonSchemaToAtom(jsonSchema.Items.Schema),
   425  			},
   426  			ElementRelationship: schema.Atomic,
   427  		}
   428  	case "integer", "number":
   429  		res.Scalar = scalarPtr(schema.Numeric)
   430  	case "string":
   431  		res.Scalar = scalarPtr(schema.String)
   432  	case "boolean":
   433  		res.Scalar = scalarPtr(schema.Boolean)
   434  	default:
   435  		panic(fmt.Sprintf("unknown JSON schema type %v", jsonSchema.Type))
   436  	}
   437  	return res
   438  }
   439  
   440  func addHierarchicalReferenceToConfig(config, spec map[string]interface{}, hierarchicalRefs []corekccv1alpha1.HierarchicalReference) (map[string]interface{}, error) {
   441  	modifiedConfig := deepcopy.MapStringInterface(config)
   442  	resourceRef, hierarchicalRef, err := GetHierarchicalReferenceFromSpec(spec, hierarchicalRefs)
   443  	if err != nil {
   444  		return nil, fmt.Errorf("error getting hierarchical reference: %v", err)
   445  	}
   446  	if resourceRef == nil {
   447  		return modifiedConfig, nil
   448  	}
   449  	var resourceRefRaw map[string]interface{}
   450  	if err := util.Marshal(resourceRef, &resourceRefRaw); err != nil {
   451  		return nil, fmt.Errorf("error marshalling hierarchical reference to map[string]interface{}: %v", err)
   452  	}
   453  	if err := unstructured.SetNestedField(modifiedConfig, resourceRefRaw, hierarchicalRef.Key); err != nil {
   454  		return nil, fmt.Errorf("error setting hierarchical reference in config: %v", err)
   455  	}
   456  	return modifiedConfig, nil
   457  }
   458
View as plain text