1
16
17 package drain
18
19 import (
20 "errors"
21 "fmt"
22
23 "github.com/spf13/cobra"
24
25 corev1 "k8s.io/api/core/v1"
26 "k8s.io/apimachinery/pkg/labels"
27 "k8s.io/apimachinery/pkg/runtime/schema"
28 utilerrors "k8s.io/apimachinery/pkg/util/errors"
29 "k8s.io/apimachinery/pkg/util/sets"
30 "k8s.io/cli-runtime/pkg/genericclioptions"
31 "k8s.io/cli-runtime/pkg/genericiooptions"
32 "k8s.io/cli-runtime/pkg/printers"
33 "k8s.io/cli-runtime/pkg/resource"
34 "k8s.io/klog/v2"
35 cmdutil "k8s.io/kubectl/pkg/cmd/util"
36 "k8s.io/kubectl/pkg/drain"
37 "k8s.io/kubectl/pkg/scheme"
38 "k8s.io/kubectl/pkg/util/completion"
39 "k8s.io/kubectl/pkg/util/i18n"
40 "k8s.io/kubectl/pkg/util/templates"
41 "k8s.io/kubectl/pkg/util/term"
42 )
43
44 type DrainCmdOptions struct {
45 PrintFlags *genericclioptions.PrintFlags
46 ToPrinter func(string) (printers.ResourcePrinterFunc, error)
47
48 Namespace string
49
50 drainer *drain.Helper
51 nodeInfos []*resource.Info
52
53 genericiooptions.IOStreams
54 WarningPrinter *printers.WarningPrinter
55 }
56
57 var (
58 cordonLong = templates.LongDesc(i18n.T(`
59 Mark node as unschedulable.`))
60
61 cordonExample = templates.Examples(i18n.T(`
62 # Mark node "foo" as unschedulable
63 kubectl cordon foo`))
64 )
65
66 func NewCmdCordon(f cmdutil.Factory, ioStreams genericiooptions.IOStreams) *cobra.Command {
67 o := NewDrainCmdOptions(f, ioStreams)
68
69 cmd := &cobra.Command{
70 Use: "cordon NODE",
71 DisableFlagsInUseLine: true,
72 Short: i18n.T("Mark node as unschedulable"),
73 Long: cordonLong,
74 Example: cordonExample,
75 ValidArgsFunction: completion.ResourceNameCompletionFunc(f, "node"),
76 Run: func(cmd *cobra.Command, args []string) {
77 cmdutil.CheckErr(o.Complete(f, cmd, args))
78 cmdutil.CheckErr(o.RunCordonOrUncordon(true))
79 },
80 }
81 cmdutil.AddLabelSelectorFlagVar(cmd, &o.drainer.Selector)
82 cmdutil.AddDryRunFlag(cmd)
83 return cmd
84 }
85
86 var (
87 uncordonLong = templates.LongDesc(i18n.T(`
88 Mark node as schedulable.`))
89
90 uncordonExample = templates.Examples(i18n.T(`
91 # Mark node "foo" as schedulable
92 kubectl uncordon foo`))
93 )
94
95 func NewCmdUncordon(f cmdutil.Factory, ioStreams genericiooptions.IOStreams) *cobra.Command {
96 o := NewDrainCmdOptions(f, ioStreams)
97
98 cmd := &cobra.Command{
99 Use: "uncordon NODE",
100 DisableFlagsInUseLine: true,
101 Short: i18n.T("Mark node as schedulable"),
102 Long: uncordonLong,
103 Example: uncordonExample,
104 ValidArgsFunction: completion.ResourceNameCompletionFunc(f, "node"),
105 Run: func(cmd *cobra.Command, args []string) {
106 cmdutil.CheckErr(o.Complete(f, cmd, args))
107 cmdutil.CheckErr(o.RunCordonOrUncordon(false))
108 },
109 }
110 cmdutil.AddLabelSelectorFlagVar(cmd, &o.drainer.Selector)
111 cmdutil.AddDryRunFlag(cmd)
112 return cmd
113 }
114
115 var (
116 drainLong = templates.LongDesc(i18n.T(`
117 Drain node in preparation for maintenance.
118
119 The given node will be marked unschedulable to prevent new pods from arriving.
120 'drain' evicts the pods if the API server supports
121 [eviction](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/). Otherwise, it will use normal
122 DELETE to delete the pods.
123 The 'drain' evicts or deletes all pods except mirror pods (which cannot be deleted through
124 the API server). If there are daemon set-managed pods, drain will not proceed
125 without --ignore-daemonsets, and regardless it will not delete any
126 daemon set-managed pods, because those pods would be immediately replaced by the
127 daemon set controller, which ignores unschedulable markings. If there are any
128 pods that are neither mirror pods nor managed by a replication controller,
129 replica set, daemon set, stateful set, or job, then drain will not delete any pods unless you
130 use --force. --force will also allow deletion to proceed if the managing resource of one
131 or more pods is missing.
132
133 'drain' waits for graceful termination. You should not operate on the machine until
134 the command completes.
135
136 When you are ready to put the node back into service, use kubectl uncordon, which
137 will make the node schedulable again.
138
139 `))
140
141 drainExample = templates.Examples(i18n.T(`
142 # Drain node "foo", even if there are pods not managed by a replication controller, replica set, job, daemon set, or stateful set on it
143 kubectl drain foo --force
144
145 # As above, but abort if there are pods not managed by a replication controller, replica set, job, daemon set, or stateful set, and use a grace period of 15 minutes
146 kubectl drain foo --grace-period=900`))
147 )
148
149 func NewDrainCmdOptions(f cmdutil.Factory, ioStreams genericiooptions.IOStreams) *DrainCmdOptions {
150 o := &DrainCmdOptions{
151 PrintFlags: genericclioptions.NewPrintFlags("drained").WithTypeSetter(scheme.Scheme),
152 IOStreams: ioStreams,
153 drainer: &drain.Helper{
154 GracePeriodSeconds: -1,
155 Out: ioStreams.Out,
156 ErrOut: ioStreams.ErrOut,
157 ChunkSize: cmdutil.DefaultChunkSize,
158 },
159 }
160 o.drainer.OnPodDeletionOrEvictionFinished = o.onPodDeletionOrEvictionFinished
161 o.drainer.OnPodDeletionOrEvictionStarted = o.onPodDeletionOrEvictionStarted
162 return o
163 }
164
165
166 func (o *DrainCmdOptions) onPodDeletionOrEvictionFinished(pod *corev1.Pod, usingEviction bool, err error) {
167 var verbStr string
168 if usingEviction {
169 if err != nil {
170 verbStr = "eviction failed"
171 } else {
172 verbStr = "evicted"
173 }
174 } else {
175 if err != nil {
176 verbStr = "deletion failed"
177 } else {
178 verbStr = "deleted"
179 }
180 }
181 printObj, err := o.ToPrinter(verbStr)
182 if err != nil {
183 fmt.Fprintf(o.ErrOut, "error building printer: %v\n", err)
184 fmt.Fprintf(o.Out, "pod %s/%s %s\n", pod.Namespace, pod.Name, verbStr)
185 } else {
186 _ = printObj(pod, o.Out)
187 }
188 }
189
190
191 func (o *DrainCmdOptions) onPodDeletionOrEvictionStarted(pod *corev1.Pod, usingEviction bool) {
192 if !klog.V(2).Enabled() {
193 return
194 }
195 var verbStr string
196 if usingEviction {
197 verbStr = "eviction started"
198 } else {
199 verbStr = "deletion started"
200 }
201 printObj, err := o.ToPrinter(verbStr)
202 if err != nil {
203 fmt.Fprintf(o.ErrOut, "error building printer: %v\n", err)
204 fmt.Fprintf(o.Out, "pod %s/%s %s\n", pod.Namespace, pod.Name, verbStr)
205 } else {
206 _ = printObj(pod, o.Out)
207 }
208 }
209
210 func NewCmdDrain(f cmdutil.Factory, ioStreams genericiooptions.IOStreams) *cobra.Command {
211 o := NewDrainCmdOptions(f, ioStreams)
212
213 cmd := &cobra.Command{
214 Use: "drain NODE",
215 DisableFlagsInUseLine: true,
216 Short: i18n.T("Drain node in preparation for maintenance"),
217 Long: drainLong,
218 Example: drainExample,
219 ValidArgsFunction: completion.ResourceNameCompletionFunc(f, "node"),
220 Run: func(cmd *cobra.Command, args []string) {
221 cmdutil.CheckErr(o.Complete(f, cmd, args))
222 cmdutil.CheckErr(o.RunDrain())
223 },
224 }
225 cmd.Flags().BoolVar(&o.drainer.Force, "force", o.drainer.Force, "Continue even if there are pods that do not declare a controller.")
226 cmd.Flags().BoolVar(&o.drainer.IgnoreAllDaemonSets, "ignore-daemonsets", o.drainer.IgnoreAllDaemonSets, "Ignore DaemonSet-managed pods.")
227 cmd.Flags().BoolVar(&o.drainer.DeleteEmptyDirData, "delete-local-data", o.drainer.DeleteEmptyDirData, "Continue even if there are pods using emptyDir (local data that will be deleted when the node is drained).")
228 cmd.Flags().MarkDeprecated("delete-local-data", "This option is deprecated and will be deleted. Use --delete-emptydir-data.")
229 cmd.Flags().BoolVar(&o.drainer.DeleteEmptyDirData, "delete-emptydir-data", o.drainer.DeleteEmptyDirData, "Continue even if there are pods using emptyDir (local data that will be deleted when the node is drained).")
230 cmd.Flags().IntVar(&o.drainer.GracePeriodSeconds, "grace-period", o.drainer.GracePeriodSeconds, "Period of time in seconds given to each pod to terminate gracefully. If negative, the default value specified in the pod will be used.")
231 cmd.Flags().DurationVar(&o.drainer.Timeout, "timeout", o.drainer.Timeout, "The length of time to wait before giving up, zero means infinite")
232 cmd.Flags().StringVarP(&o.drainer.PodSelector, "pod-selector", "", o.drainer.PodSelector, "Label selector to filter pods on the node")
233 cmd.Flags().BoolVar(&o.drainer.DisableEviction, "disable-eviction", o.drainer.DisableEviction, "Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution.")
234 cmd.Flags().IntVar(&o.drainer.SkipWaitForDeleteTimeoutSeconds, "skip-wait-for-delete-timeout", o.drainer.SkipWaitForDeleteTimeoutSeconds, "If pod DeletionTimestamp older than N seconds, skip waiting for the pod. Seconds must be greater than 0 to skip.")
235
236 cmdutil.AddChunkSizeFlag(cmd, &o.drainer.ChunkSize)
237 cmdutil.AddDryRunFlag(cmd)
238 cmdutil.AddLabelSelectorFlagVar(cmd, &o.drainer.Selector)
239 return cmd
240 }
241
242
243
244 func (o *DrainCmdOptions) Complete(f cmdutil.Factory, cmd *cobra.Command, args []string) error {
245 var err error
246
247 if len(args) == 0 && !cmd.Flags().Changed("selector") {
248 return cmdutil.UsageErrorf(cmd, fmt.Sprintf("USAGE: %s [flags]", cmd.Use))
249 }
250 if len(args) > 0 && len(o.drainer.Selector) > 0 {
251 return cmdutil.UsageErrorf(cmd, "error: cannot specify both a node name and a --selector option")
252 }
253
254 o.drainer.DryRunStrategy, err = cmdutil.GetDryRunStrategy(cmd)
255 if err != nil {
256 return err
257 }
258
259 if o.drainer.Client, err = f.KubernetesClientSet(); err != nil {
260 return err
261 }
262
263 if len(o.drainer.PodSelector) > 0 {
264 if _, err := labels.Parse(o.drainer.PodSelector); err != nil {
265 return errors.New("--pod-selector=<pod_selector> must be a valid label selector")
266 }
267 }
268
269 o.nodeInfos = []*resource.Info{}
270
271 o.Namespace, _, err = f.ToRawKubeConfigLoader().Namespace()
272 if err != nil {
273 return err
274 }
275
276 o.ToPrinter = func(operation string) (printers.ResourcePrinterFunc, error) {
277 o.PrintFlags.NamePrintFlags.Operation = operation
278 cmdutil.PrintFlagsWithDryRunStrategy(o.PrintFlags, o.drainer.DryRunStrategy)
279
280 printer, err := o.PrintFlags.ToPrinter()
281 if err != nil {
282 return nil, err
283 }
284
285 return printer.PrintObj, nil
286 }
287
288
289 if o.WarningPrinter == nil {
290 o.WarningPrinter = printers.NewWarningPrinter(o.ErrOut, printers.WarningPrinterOptions{Color: term.AllowsColorOutput(o.ErrOut)})
291 }
292
293 builder := f.NewBuilder().
294 WithScheme(scheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...).
295 NamespaceParam(o.Namespace).DefaultNamespace().
296 RequestChunksOf(o.drainer.ChunkSize).
297 ResourceNames("nodes", args...).
298 SingleResourceType().
299 Flatten()
300
301 if len(o.drainer.Selector) > 0 {
302 builder = builder.LabelSelectorParam(o.drainer.Selector).
303 ResourceTypes("nodes")
304 }
305
306 r := builder.Do()
307
308 if err = r.Err(); err != nil {
309 return err
310 }
311
312 return r.Visit(func(info *resource.Info, err error) error {
313 if err != nil {
314 return err
315 }
316 if info.Mapping.Resource.GroupResource() != (schema.GroupResource{Group: "", Resource: "nodes"}) {
317 return fmt.Errorf("error: expected resource of type node, got %q", info.Mapping.Resource)
318 }
319
320 o.nodeInfos = append(o.nodeInfos, info)
321 return nil
322 })
323 }
324
325
326 func (o *DrainCmdOptions) RunDrain() error {
327 if err := o.RunCordonOrUncordon(true); err != nil {
328 return err
329 }
330
331 drainedNodes := sets.NewString()
332 var fatal []error
333
334 remainingNodes := []string{}
335 for _, info := range o.nodeInfos {
336 if err := o.deleteOrEvictPodsSimple(info); err == nil {
337 drainedNodes.Insert(info.Name)
338
339 printObj, err := o.ToPrinter("drained")
340 if err != nil {
341 return err
342 }
343
344 printObj(info.Object, o.Out)
345 } else {
346 fmt.Fprintf(o.ErrOut, "error: unable to drain node %q due to error:%s, continuing command...\n", info.Name, err)
347
348 if !drainedNodes.Has(info.Name) {
349 fatal = append(fatal, err)
350 remainingNodes = append(remainingNodes, info.Name)
351 }
352
353 continue
354 }
355 }
356
357 if len(remainingNodes) > 0 {
358 fmt.Fprintf(o.ErrOut, "There are pending nodes to be drained:\n")
359 for _, nodeName := range remainingNodes {
360 fmt.Fprintf(o.ErrOut, " %s\n", nodeName)
361 }
362 }
363
364 return utilerrors.NewAggregate(fatal)
365 }
366
367 func (o *DrainCmdOptions) deleteOrEvictPodsSimple(nodeInfo *resource.Info) error {
368 list, errs := o.drainer.GetPodsForDeletion(nodeInfo.Name)
369 if errs != nil {
370 return utilerrors.NewAggregate(errs)
371 }
372 if warnings := list.Warnings(); warnings != "" {
373 o.WarningPrinter.Print(warnings)
374 }
375 if o.drainer.DryRunStrategy == cmdutil.DryRunClient {
376 for _, pod := range list.Pods() {
377 fmt.Fprintf(o.Out, "evicting pod %s/%s (dry run)\n", pod.Namespace, pod.Name)
378 }
379 return nil
380 }
381
382 if err := o.drainer.DeleteOrEvictPods(list.Pods()); err != nil {
383 pendingList, newErrs := o.drainer.GetPodsForDeletion(nodeInfo.Name)
384 if pendingList != nil {
385 pods := pendingList.Pods()
386 if len(pods) != 0 {
387 fmt.Fprintf(o.ErrOut, "There are pending pods in node %q when an error occurred: %v\n", nodeInfo.Name, err)
388 for _, pendingPod := range pods {
389 fmt.Fprintf(o.ErrOut, "%s/%s\n", "pod", pendingPod.Name)
390 }
391 }
392 }
393 if newErrs != nil {
394 fmt.Fprintf(o.ErrOut, "Following errors occurred while getting the list of pods to delete:\n%s", utilerrors.NewAggregate(newErrs))
395 }
396 return err
397 }
398 return nil
399 }
400
401
402
403 func (o *DrainCmdOptions) RunCordonOrUncordon(desired bool) error {
404 cordonOrUncordon := "cordon"
405 if !desired {
406 cordonOrUncordon = "un" + cordonOrUncordon
407 }
408
409 for _, nodeInfo := range o.nodeInfos {
410
411 printError := func(err error) {
412 fmt.Fprintf(o.ErrOut, "error: unable to %s node %q: %v\n", cordonOrUncordon, nodeInfo.Name, err)
413 }
414
415 gvk := nodeInfo.ResourceMapping().GroupVersionKind
416 if gvk.Kind == "Node" {
417 c, err := drain.NewCordonHelperFromRuntimeObject(nodeInfo.Object, scheme.Scheme, gvk)
418 if err != nil {
419 printError(err)
420 continue
421 }
422
423 if updateRequired := c.UpdateIfRequired(desired); !updateRequired {
424 printObj, err := o.ToPrinter(already(desired))
425 if err != nil {
426 fmt.Fprintf(o.ErrOut, "error: %v\n", err)
427 continue
428 }
429 printObj(nodeInfo.Object, o.Out)
430 } else {
431 if o.drainer.DryRunStrategy != cmdutil.DryRunClient {
432 err, patchErr := c.PatchOrReplace(o.drainer.Client, o.drainer.DryRunStrategy == cmdutil.DryRunServer)
433 if patchErr != nil {
434 printError(patchErr)
435 }
436 if err != nil {
437 printError(err)
438 continue
439 }
440 }
441 printObj, err := o.ToPrinter(changed(desired))
442 if err != nil {
443 fmt.Fprintf(o.ErrOut, "%v\n", err)
444 continue
445 }
446 printObj(nodeInfo.Object, o.Out)
447 }
448 } else {
449 printObj, err := o.ToPrinter("skipped")
450 if err != nil {
451 fmt.Fprintf(o.ErrOut, "%v\n", err)
452 continue
453 }
454 printObj(nodeInfo.Object, o.Out)
455 }
456 }
457
458 return nil
459 }
460
461
462
463 func already(desired bool) string {
464 if desired {
465 return "already cordoned"
466 }
467 return "already uncordoned"
468 }
469
470 func changed(desired bool) string {
471 if desired {
472 return "cordoned"
473 }
474 return "uncordoned"
475 }
476
View as plain text