1
16
17 package apply
18
19 import (
20 "context"
21 "crypto/sha256"
22 "encoding/base64"
23 "encoding/json"
24 "fmt"
25 "sort"
26 "strings"
27
28 "k8s.io/apimachinery/pkg/api/errors"
29 "k8s.io/apimachinery/pkg/api/meta"
30 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
31 "k8s.io/apimachinery/pkg/runtime"
32 "k8s.io/apimachinery/pkg/runtime/schema"
33 "k8s.io/apimachinery/pkg/types"
34 utilerrors "k8s.io/apimachinery/pkg/util/errors"
35 "k8s.io/apimachinery/pkg/util/sets"
36 "k8s.io/cli-runtime/pkg/resource"
37 "k8s.io/client-go/dynamic"
38 "k8s.io/klog/v2"
39 cmdutil "k8s.io/kubectl/pkg/cmd/util"
40 )
41
42
43
44 const (
45
46
47
48
49 ApplySetToolingAnnotation = "applyset.kubernetes.io/tooling"
50
51
52
53
54
55 ApplySetAdditionalNamespacesAnnotation = "applyset.kubernetes.io/additional-namespaces"
56
57
58
59
60
61
62
63
64 DeprecatedApplySetGRsAnnotation = "applyset.kubernetes.io/contains-group-resources"
65
66
67
68
69
70
71
72 ApplySetGKsAnnotation = "applyset.kubernetes.io/contains-group-kinds"
73
74
75
76 ApplySetParentIDLabel = "applyset.kubernetes.io/id"
77
78
79
80
81
82 V1ApplySetIdFormat = "applyset-%s-v1"
83
84
85
86 ApplysetPartOfLabel = "applyset.kubernetes.io/part-of"
87
88
89
90 ApplysetParentCRDLabel = "applyset.kubernetes.io/is-parent-type"
91 )
92
93 var defaultApplySetParentGVR = schema.GroupVersionResource{Version: "v1", Resource: "secrets"}
94
95
96 type ApplySet struct {
97
98 parentRef *ApplySetParentRef
99
100
101 toolingID ApplySetTooling
102
103
104 currentResources map[schema.GroupKind]*kindInfo
105
106
107 currentNamespaces sets.Set[string]
108
109
110 updatedResources map[schema.GroupKind]*kindInfo
111
112
113 updatedNamespaces sets.Set[string]
114
115 restMapper meta.RESTMapper
116
117
118 client resource.RESTClient
119 }
120
121 var builtinApplySetParentGVRs = sets.New[schema.GroupVersionResource](
122 defaultApplySetParentGVR,
123 schema.GroupVersionResource{Version: "v1", Resource: "configmaps"},
124 )
125
126
127 type ApplySetParentRef struct {
128 Name string
129 Namespace string
130 *meta.RESTMapping
131 }
132
133 func (p ApplySetParentRef) IsNamespaced() bool {
134 return p.Scope.Name() == meta.RESTScopeNameNamespace
135 }
136
137
138
139 func (p ApplySetParentRef) String() string {
140 return fmt.Sprintf("%s.%s/%s", p.Resource.Resource, p.Resource.Group, p.Name)
141 }
142
143 type ApplySetTooling struct {
144 Name string
145 Version string
146 }
147
148 func (t ApplySetTooling) String() string {
149 return fmt.Sprintf("%s/%s", t.Name, t.Version)
150 }
151
152
153 func NewApplySet(parent *ApplySetParentRef, tooling ApplySetTooling, mapper meta.RESTMapper, client resource.RESTClient) *ApplySet {
154 return &ApplySet{
155 currentResources: make(map[schema.GroupKind]*kindInfo),
156 currentNamespaces: make(sets.Set[string]),
157 updatedResources: make(map[schema.GroupKind]*kindInfo),
158 updatedNamespaces: make(sets.Set[string]),
159 parentRef: parent,
160 toolingID: tooling,
161 restMapper: mapper,
162 client: client,
163 }
164 }
165
166 const applySetIDPartDelimiter = "."
167
168
169
170
171 func (a ApplySet) ID() string {
172 unencoded := strings.Join([]string{a.parentRef.Name, a.parentRef.Namespace, a.parentRef.GroupVersionKind.Kind, a.parentRef.GroupVersionKind.Group}, applySetIDPartDelimiter)
173 hashed := sha256.Sum256([]byte(unencoded))
174 b64 := base64.RawURLEncoding.EncodeToString(hashed[:])
175
176 return fmt.Sprintf(V1ApplySetIdFormat, b64)
177 }
178
179
180 func (a ApplySet) Validate(ctx context.Context, client dynamic.Interface) error {
181 var errors []error
182 if a.parentRef.IsNamespaced() && a.parentRef.Namespace == "" {
183 errors = append(errors, fmt.Errorf("namespace is required to use namespace-scoped ApplySet"))
184 }
185 if !builtinApplySetParentGVRs.Has(a.parentRef.Resource) {
186
187
188 permittedCRParents, err := a.getAllowedCustomResourceParents(ctx, client)
189 if err != nil {
190 errors = append(errors, fmt.Errorf("identifying allowed custom resource parent types: %w", err))
191 }
192 parentRefResourceIgnoreVersion := a.parentRef.Resource.GroupResource().WithVersion("")
193 if !permittedCRParents.Has(parentRefResourceIgnoreVersion) {
194 errors = append(errors, fmt.Errorf("resource %q is not permitted as an ApplySet parent", a.parentRef.Resource))
195 }
196 }
197 return utilerrors.NewAggregate(errors)
198 }
199
200 func (a *ApplySet) labelForCustomParentCRDs() *metav1.LabelSelector {
201 return &metav1.LabelSelector{
202 MatchExpressions: []metav1.LabelSelectorRequirement{{
203 Key: ApplysetParentCRDLabel,
204 Operator: metav1.LabelSelectorOpExists,
205 }},
206 }
207 }
208
209 func (a *ApplySet) getAllowedCustomResourceParents(ctx context.Context, client dynamic.Interface) (sets.Set[schema.GroupVersionResource], error) {
210 opts := metav1.ListOptions{
211 LabelSelector: metav1.FormatLabelSelector(a.labelForCustomParentCRDs()),
212 }
213 list, err := client.Resource(schema.GroupVersionResource{
214 Group: "apiextensions.k8s.io",
215 Version: "v1",
216 Resource: "customresourcedefinitions",
217 }).List(ctx, opts)
218 if err != nil {
219 return nil, err
220 }
221 set := sets.New[schema.GroupVersionResource]()
222 for i := range list.Items {
223
224
225 gr := schema.ParseGroupResource(list.Items[i].GetName())
226 set.Insert(gr.WithVersion(""))
227 }
228 return set, nil
229 }
230
231 func (a *ApplySet) LabelsForMember() map[string]string {
232 return map[string]string{
233 ApplysetPartOfLabel: a.ID(),
234 }
235 }
236
237
238 func (a *ApplySet) AddLabels(objects ...*resource.Info) error {
239 applysetLabels := a.LabelsForMember()
240 for _, obj := range objects {
241 accessor, err := meta.Accessor(obj.Object)
242 if err != nil {
243 return fmt.Errorf("getting accessor: %w", err)
244 }
245 labels := accessor.GetLabels()
246 if labels == nil {
247 labels = make(map[string]string)
248 }
249 for k, v := range applysetLabels {
250 if _, found := labels[k]; found {
251 return fmt.Errorf("ApplySet label %q already set in input data", k)
252 }
253 labels[k] = v
254 }
255 accessor.SetLabels(labels)
256 }
257
258 return nil
259 }
260
261 func (a *ApplySet) fetchParent() error {
262 helper := resource.NewHelper(a.client, a.parentRef.RESTMapping)
263 obj, err := helper.Get(a.parentRef.Namespace, a.parentRef.Name)
264 if errors.IsNotFound(err) {
265 if !builtinApplySetParentGVRs.Has(a.parentRef.Resource) {
266 return fmt.Errorf("custom resource ApplySet parents cannot be created automatically")
267 }
268 return nil
269 } else if err != nil {
270 return fmt.Errorf("failed to fetch ApplySet parent object %q: %w", a.parentRef, err)
271 } else if obj == nil {
272 return fmt.Errorf("failed to fetch ApplySet parent object %q", a.parentRef)
273 }
274
275 labels, annotations, err := getLabelsAndAnnotations(obj)
276 if err != nil {
277 return fmt.Errorf("getting metadata from parent object %q: %w", a.parentRef, err)
278 }
279
280 toolAnnotation, hasToolAnno := annotations[ApplySetToolingAnnotation]
281 if !hasToolAnno {
282 return fmt.Errorf("ApplySet parent object %q already exists and is missing required annotation %q", a.parentRef, ApplySetToolingAnnotation)
283 }
284 if managedBy := toolingBaseName(toolAnnotation); managedBy != a.toolingID.Name {
285 return fmt.Errorf("ApplySet parent object %q already exists and is managed by tooling %q instead of %q", a.parentRef, managedBy, a.toolingID.Name)
286 }
287
288 idLabel, hasIDLabel := labels[ApplySetParentIDLabel]
289 if !hasIDLabel {
290 return fmt.Errorf("ApplySet parent object %q exists and does not have required label %s", a.parentRef, ApplySetParentIDLabel)
291 }
292 if idLabel != a.ID() {
293 return fmt.Errorf("ApplySet parent object %q exists and has incorrect value for label %q (got: %s, want: %s)", a.parentRef, ApplySetParentIDLabel, idLabel, a.ID())
294 }
295
296 if a.currentResources, err = parseKindAnnotation(annotations, a.restMapper); err != nil {
297
298 return fmt.Errorf("parsing ApplySet annotation on %q: %w", a.parentRef, err)
299 }
300 a.currentNamespaces = parseNamespacesAnnotation(annotations)
301 if a.parentRef.IsNamespaced() {
302 a.currentNamespaces.Insert(a.parentRef.Namespace)
303 }
304 return nil
305 }
306 func (a *ApplySet) LabelSelectorForMembers() string {
307 return metav1.FormatLabelSelector(&metav1.LabelSelector{
308 MatchLabels: a.LabelsForMember(),
309 })
310 }
311
312
313
314 func (a *ApplySet) AllPrunableResources() []*kindInfo {
315 var ret []*kindInfo
316 for _, m := range a.currentResources {
317 ret = append(ret, m)
318 }
319 return ret
320 }
321
322
323
324 func (a *ApplySet) AllPrunableNamespaces() []string {
325 var ret []string
326 for ns := range a.currentNamespaces {
327 ret = append(ret, ns)
328 }
329 return ret
330 }
331
332 func getLabelsAndAnnotations(obj runtime.Object) (map[string]string, map[string]string, error) {
333 accessor, err := meta.Accessor(obj)
334 if err != nil {
335 return nil, nil, err
336 }
337 return accessor.GetLabels(), accessor.GetAnnotations(), nil
338 }
339
340 func toolingBaseName(toolAnnotation string) string {
341 parts := strings.Split(toolAnnotation, "/")
342 if len(parts) >= 2 {
343 return strings.Join(parts[:len(parts)-1], "/")
344 }
345 return toolAnnotation
346 }
347
348
349 type kindInfo struct {
350 restMapping *meta.RESTMapping
351 }
352
353 func parseKindAnnotation(annotations map[string]string, mapper meta.RESTMapper) (map[schema.GroupKind]*kindInfo, error) {
354 annotation, ok := annotations[ApplySetGKsAnnotation]
355 if !ok {
356 if annotations[DeprecatedApplySetGRsAnnotation] != "" {
357 return parseDeprecatedResourceAnnotation(annotations[DeprecatedApplySetGRsAnnotation], mapper)
358 }
359
360
361
362 return nil, fmt.Errorf("kubectl requires the %q annotation to be set on all ApplySet parent objects", ApplySetGKsAnnotation)
363 }
364 mappings := make(map[schema.GroupKind]*kindInfo)
365
366 if annotation == "" {
367 return mappings, nil
368 }
369 for _, gkString := range strings.Split(annotation, ",") {
370 gk := schema.ParseGroupKind(gkString)
371 restMapping, err := mapper.RESTMapping(gk)
372 if err != nil {
373 return nil, fmt.Errorf("could not find mapping for kind in %q annotation: %w", ApplySetGKsAnnotation, err)
374 }
375 mappings[gk] = &kindInfo{
376 restMapping: restMapping,
377 }
378 }
379
380 return mappings, nil
381 }
382
383 func parseDeprecatedResourceAnnotation(annotation string, mapper meta.RESTMapper) (map[schema.GroupKind]*kindInfo, error) {
384 mappings := make(map[schema.GroupKind]*kindInfo)
385
386 if annotation == "" {
387 return mappings, nil
388 }
389 for _, grString := range strings.Split(annotation, ",") {
390 gr := schema.ParseGroupResource(grString)
391 gvk, err := mapper.KindFor(gr.WithVersion(""))
392 if err != nil {
393 return nil, fmt.Errorf("invalid group resource in %q annotation: %w", DeprecatedApplySetGRsAnnotation, err)
394 }
395 restMapping, err := mapper.RESTMapping(gvk.GroupKind())
396 if err != nil {
397 return nil, fmt.Errorf("could not find kind for resource in %q annotation: %w", DeprecatedApplySetGRsAnnotation, err)
398 }
399 mappings[gvk.GroupKind()] = &kindInfo{
400 restMapping: restMapping,
401 }
402 }
403 return mappings, nil
404 }
405
406 func parseNamespacesAnnotation(annotations map[string]string) sets.Set[string] {
407 annotation, ok := annotations[ApplySetAdditionalNamespacesAnnotation]
408 if !ok {
409 return sets.Set[string]{}
410 }
411
412 if annotation == "" {
413 return sets.Set[string]{}
414 }
415 return sets.New(strings.Split(annotation, ",")...)
416 }
417
418
419
420 func (a *ApplySet) addResource(restMapping *meta.RESTMapping, namespace string) {
421 gk := restMapping.GroupVersionKind.GroupKind()
422 if _, found := a.updatedResources[gk]; !found {
423 a.updatedResources[gk] = &kindInfo{
424 restMapping: restMapping,
425 }
426 }
427 if restMapping.Scope == meta.RESTScopeNamespace && namespace != "" {
428 a.updatedNamespaces.Insert(namespace)
429 }
430 }
431
432 type ApplySetUpdateMode string
433
434 var updateToLatestSet ApplySetUpdateMode = "latest"
435 var updateToSuperset ApplySetUpdateMode = "superset"
436
437 func (a *ApplySet) updateParent(mode ApplySetUpdateMode, dryRun cmdutil.DryRunStrategy, validation string) error {
438 data, err := json.Marshal(a.buildParentPatch(mode))
439 if err != nil {
440 return fmt.Errorf("failed to encode patch for ApplySet parent: %w", err)
441 }
442
443
444 err = serverSideApplyRequest(a, data, dryRun, validation, false)
445 if err != nil && errors.IsConflict(err) {
446
447 klog.Warningf("WARNING: failed to update ApplySet: %s\nApplySet field manager %s should own these fields. Retrying with conflicts forced.", err.Error(), a.FieldManager())
448 err = serverSideApplyRequest(a, data, dryRun, validation, true)
449 }
450 if err != nil {
451 return fmt.Errorf("failed to update ApplySet: %w", err)
452 }
453 return nil
454 }
455
456 func serverSideApplyRequest(a *ApplySet, data []byte, dryRun cmdutil.DryRunStrategy, validation string, forceConficts bool) error {
457 if dryRun == cmdutil.DryRunClient {
458 return nil
459 }
460 helper := resource.NewHelper(a.client, a.parentRef.RESTMapping).
461 DryRun(dryRun == cmdutil.DryRunServer).
462 WithFieldManager(a.FieldManager()).
463 WithFieldValidation(validation)
464
465 options := metav1.PatchOptions{
466 Force: &forceConficts,
467 }
468 _, err := helper.Patch(
469 a.parentRef.Namespace,
470 a.parentRef.Name,
471 types.ApplyPatchType,
472 data,
473 &options,
474 )
475 return err
476 }
477
478 func (a *ApplySet) buildParentPatch(mode ApplySetUpdateMode) *metav1.PartialObjectMetadata {
479 var newGKsAnnotation, newNsAnnotation string
480 switch mode {
481 case updateToSuperset:
482
483
484
485 grSuperset := sets.KeySet(a.currentResources).Union(sets.KeySet(a.updatedResources))
486 newGKsAnnotation = generateKindsAnnotation(grSuperset)
487 newNsAnnotation = generateNamespacesAnnotation(a.currentNamespaces.Union(a.updatedNamespaces), a.parentRef.Namespace)
488 case updateToLatestSet:
489 newGKsAnnotation = generateKindsAnnotation(sets.KeySet(a.updatedResources))
490 newNsAnnotation = generateNamespacesAnnotation(a.updatedNamespaces, a.parentRef.Namespace)
491 }
492
493 return &metav1.PartialObjectMetadata{
494 TypeMeta: metav1.TypeMeta{
495 Kind: a.parentRef.GroupVersionKind.Kind,
496 APIVersion: a.parentRef.GroupVersionKind.GroupVersion().String(),
497 },
498 ObjectMeta: metav1.ObjectMeta{
499 Name: a.parentRef.Name,
500 Namespace: a.parentRef.Namespace,
501 Annotations: map[string]string{
502 ApplySetToolingAnnotation: a.toolingID.String(),
503 ApplySetGKsAnnotation: newGKsAnnotation,
504 ApplySetAdditionalNamespacesAnnotation: newNsAnnotation,
505 },
506 Labels: map[string]string{
507 ApplySetParentIDLabel: a.ID(),
508 },
509 },
510 }
511 }
512
513 func generateNamespacesAnnotation(namespaces sets.Set[string], skip string) string {
514 nsList := namespaces.Clone().Delete(skip).UnsortedList()
515 sort.Strings(nsList)
516 return strings.Join(nsList, ",")
517 }
518
519 func generateKindsAnnotation(resources sets.Set[schema.GroupKind]) string {
520 var gks []string
521 for gk := range resources {
522 gks = append(gks, gk.String())
523 }
524 sort.Strings(gks)
525 return strings.Join(gks, ",")
526 }
527
528 func (a ApplySet) FieldManager() string {
529 return fmt.Sprintf("%s-applyset", a.toolingID.Name)
530 }
531
532
533 func ParseApplySetParentRef(parentRefStr string, mapper meta.RESTMapper) (*ApplySetParentRef, error) {
534 var gvr schema.GroupVersionResource
535 var name string
536
537 if groupRes, nameSuffix, hasTypeInfo := strings.Cut(parentRefStr, "/"); hasTypeInfo {
538 name = nameSuffix
539 gvr = schema.ParseGroupResource(groupRes).WithVersion("")
540 } else {
541 name = parentRefStr
542 gvr = defaultApplySetParentGVR
543 }
544
545 if name == "" {
546 return nil, fmt.Errorf("name cannot be blank")
547 }
548
549 gvk, err := mapper.KindFor(gvr)
550 if err != nil {
551 return nil, err
552 }
553 mapping, err := mapper.RESTMapping(gvk.GroupKind())
554 if err != nil {
555 return nil, err
556 }
557 return &ApplySetParentRef{Name: name, RESTMapping: mapping}, nil
558 }
559
560
561 func (a *ApplySet) Prune(ctx context.Context, o *ApplyOptions) error {
562 printer, err := o.ToPrinter("pruned")
563 if err != nil {
564 return err
565 }
566 opt := &ApplySetDeleteOptions{
567 CascadingStrategy: o.DeleteOptions.CascadingStrategy,
568 DryRunStrategy: o.DryRunStrategy,
569 GracePeriod: o.DeleteOptions.GracePeriod,
570
571 Printer: printer,
572
573 IOStreams: o.IOStreams,
574 }
575
576 if err := a.pruneAll(ctx, o.DynamicClient, o.VisitedUids, opt); err != nil {
577 return err
578 }
579
580 if err := a.updateParent(updateToLatestSet, o.DryRunStrategy, o.ValidationDirective); err != nil {
581 return fmt.Errorf("apply and prune succeeded, but ApplySet update failed: %w", err)
582 }
583
584 return nil
585 }
586
587
588
589
590 func (a *ApplySet) BeforeApply(objects []*resource.Info, dryRunStrategy cmdutil.DryRunStrategy, validationDirective string) error {
591 if err := a.fetchParent(); err != nil {
592 return err
593 }
594
595
596
597
598
599
600 for _, info := range objects {
601 a.addResource(info.ResourceMapping(), info.Namespace)
602 }
603 if err := a.updateParent(updateToSuperset, dryRunStrategy, validationDirective); err != nil {
604 return err
605 }
606 return nil
607 }
608
View as plain text