debug.go

Documentation: k8s.io/kubectl/pkg/cmd/debug

     1  /*
     2  Copyright 2020 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package debug
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"fmt"
    23  	"os"
    24  	"time"
    25  
    26  	"github.com/distribution/reference"
    27  	"github.com/spf13/cobra"
    28  	"k8s.io/klog/v2"
    29  	"k8s.io/utils/pointer"
    30  
    31  	corev1 "k8s.io/api/core/v1"
    32  	"k8s.io/apimachinery/pkg/api/errors"
    33  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    34  	"k8s.io/apimachinery/pkg/fields"
    35  	"k8s.io/apimachinery/pkg/runtime"
    36  	"k8s.io/apimachinery/pkg/runtime/schema"
    37  	"k8s.io/apimachinery/pkg/types"
    38  	utilrand "k8s.io/apimachinery/pkg/util/rand"
    39  	"k8s.io/apimachinery/pkg/util/strategicpatch"
    40  	"k8s.io/apimachinery/pkg/watch"
    41  	"k8s.io/cli-runtime/pkg/genericclioptions"
    42  	"k8s.io/cli-runtime/pkg/genericiooptions"
    43  	"k8s.io/cli-runtime/pkg/printers"
    44  	"k8s.io/cli-runtime/pkg/resource"
    45  	"k8s.io/client-go/kubernetes"
    46  	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
    47  	"k8s.io/client-go/tools/cache"
    48  	watchtools "k8s.io/client-go/tools/watch"
    49  	"k8s.io/kubectl/pkg/cmd/attach"
    50  	"k8s.io/kubectl/pkg/cmd/exec"
    51  	"k8s.io/kubectl/pkg/cmd/logs"
    52  	cmdutil "k8s.io/kubectl/pkg/cmd/util"
    53  	"k8s.io/kubectl/pkg/polymorphichelpers"
    54  	"k8s.io/kubectl/pkg/scheme"
    55  	"k8s.io/kubectl/pkg/util/i18n"
    56  	"k8s.io/kubectl/pkg/util/interrupt"
    57  	"k8s.io/kubectl/pkg/util/templates"
    58  	"k8s.io/kubectl/pkg/util/term"
    59  )
    60  
    61  var (
    62  	debugLong = templates.LongDesc(i18n.T(`
    63  		Debug cluster resources using interactive debugging containers.
    64  
    65  		'debug' provides automation for common debugging tasks for cluster objects identified by
    66  		resource and name. Pods will be used by default if no resource is specified.
    67  
    68  		The action taken by 'debug' varies depending on what resource is specified. Supported
    69  		actions include:
    70  
    71  		* Workload: Create a copy of an existing pod with certain attributes changed,
    72  	                for example changing the image tag to a new version.
    73  		* Workload: Add an ephemeral container to an already running pod, for example to add
    74  		            debugging utilities without restarting the pod.
    75  		* Node: Create a new pod that runs in the node's host namespaces and can access
    76  		        the node's filesystem.
    77  `))
    78  
    79  	debugExample = templates.Examples(i18n.T(`
    80  		# Create an interactive debugging session in pod mypod and immediately attach to it.
    81  		kubectl debug mypod -it --image=busybox
    82  
    83  		# Create an interactive debugging session for the pod in the file pod.yaml and immediately attach to it.
    84  		# (requires the EphemeralContainers feature to be enabled in the cluster)
    85  		kubectl debug -f pod.yaml -it --image=busybox
    86  
    87  		# Create a debug container named debugger using a custom automated debugging image.
    88  		kubectl debug --image=myproj/debug-tools -c debugger mypod
    89  
    90  		# Create a copy of mypod adding a debug container and attach to it
    91  		kubectl debug mypod -it --image=busybox --copy-to=my-debugger
    92  
    93  		# Create a copy of mypod changing the command of mycontainer
    94  		kubectl debug mypod -it --copy-to=my-debugger --container=mycontainer -- sh
    95  
    96  		# Create a copy of mypod changing all container images to busybox
    97  		kubectl debug mypod --copy-to=my-debugger --set-image=*=busybox
    98  
    99  		# Create a copy of mypod adding a debug container and changing container images
   100  		kubectl debug mypod -it --copy-to=my-debugger --image=debian --set-image=app=app:debug,sidecar=sidecar:debug
   101  
   102  		# Create an interactive debugging session on a node and immediately attach to it.
   103  		# The container will run in the host namespaces and the host's filesystem will be mounted at /host
   104  		kubectl debug node/mynode -it --image=busybox
   105  `))
   106  )
   107  
   108  var nameSuffixFunc = utilrand.String
   109  
   110  type DebugAttachFunc func(ctx context.Context, restClientGetter genericclioptions.RESTClientGetter, cmdPath string, ns, podName, containerName string) error
   111  
   112  // DebugOptions holds the options for an invocation of kubectl debug.
   113  type DebugOptions struct {
   114  	Args              []string
   115  	ArgsOnly          bool
   116  	Attach            bool
   117  	AttachFunc        DebugAttachFunc
   118  	Container         string
   119  	CopyTo            string
   120  	Replace           bool
   121  	Env               []corev1.EnvVar
   122  	Image             string
   123  	Interactive       bool
   124  	Namespace         string
   125  	TargetNames       []string
   126  	PullPolicy        corev1.PullPolicy
   127  	Quiet             bool
   128  	SameNode          bool
   129  	SetImages         map[string]string
   130  	ShareProcesses    bool
   131  	TargetContainer   string
   132  	TTY               bool
   133  	Profile           string
   134  	CustomProfileFile string
   135  	CustomProfile     *corev1.Container
   136  	Applier           ProfileApplier
   137  
   138  	explicitNamespace     bool
   139  	attachChanged         bool
   140  	shareProcessedChanged bool
   141  
   142  	podClient corev1client.CoreV1Interface
   143  
   144  	Builder *resource.Builder
   145  	genericiooptions.IOStreams
   146  	WarningPrinter *printers.WarningPrinter
   147  
   148  	resource.FilenameOptions
   149  }
   150  
   151  // NewDebugOptions returns a DebugOptions initialized with default values.
   152  func NewDebugOptions(streams genericiooptions.IOStreams) *DebugOptions {
   153  	return &DebugOptions{
   154  		Args:           []string{},
   155  		IOStreams:      streams,
   156  		TargetNames:    []string{},
   157  		ShareProcesses: true,
   158  	}
   159  }
   160  
   161  // NewCmdDebug returns a cobra command that runs kubectl debug.
   162  func NewCmdDebug(restClientGetter genericclioptions.RESTClientGetter, streams genericiooptions.IOStreams) *cobra.Command {
   163  	o := NewDebugOptions(streams)
   164  
   165  	cmd := &cobra.Command{
   166  		Use:                   "debug (POD | TYPE[[.VERSION].GROUP]/NAME) [ -- COMMAND [args...] ]",
   167  		DisableFlagsInUseLine: true,
   168  		Short:                 i18n.T("Create debugging sessions for troubleshooting workloads and nodes"),
   169  		Long:                  debugLong,
   170  		Example:               debugExample,
   171  		Run: func(cmd *cobra.Command, args []string) {
   172  			cmdutil.CheckErr(o.Complete(restClientGetter, cmd, args))
   173  			cmdutil.CheckErr(o.Validate())
   174  			cmdutil.CheckErr(o.Run(restClientGetter, cmd))
   175  		},
   176  	}
   177  
   178  	o.AddFlags(cmd)
   179  	return cmd
   180  }
   181  
   182  func (o *DebugOptions) AddFlags(cmd *cobra.Command) {
   183  	cmdutil.AddJsonFilenameFlag(cmd.Flags(), &o.FilenameOptions.Filenames, "identifying the resource to debug")
   184  
   185  	cmd.Flags().BoolVar(&o.ArgsOnly, "arguments-only", o.ArgsOnly, i18n.T("If specified, everything after -- will be passed to the new container as Args instead of Command."))
   186  	cmd.Flags().BoolVar(&o.Attach, "attach", o.Attach, i18n.T("If true, wait for the container to start running, and then attach as if 'kubectl attach ...' were called.  Default false, unless '-i/--stdin' is set, in which case the default is true."))
   187  	cmd.Flags().StringVarP(&o.Container, "container", "c", o.Container, i18n.T("Container name to use for debug container."))
   188  	cmd.Flags().StringVar(&o.CopyTo, "copy-to", o.CopyTo, i18n.T("Create a copy of the target Pod with this name."))
   189  	cmd.Flags().BoolVar(&o.Replace, "replace", o.Replace, i18n.T("When used with '--copy-to', delete the original Pod."))
   190  	cmd.Flags().StringToString("env", nil, i18n.T("Environment variables to set in the container."))
   191  	cmd.Flags().StringVar(&o.Image, "image", o.Image, i18n.T("Container image to use for debug container."))
   192  	cmd.Flags().StringToStringVar(&o.SetImages, "set-image", o.SetImages, i18n.T("When used with '--copy-to', a list of name=image pairs for changing container images, similar to how 'kubectl set image' works."))
   193  	cmd.Flags().String("image-pull-policy", "", i18n.T("The image pull policy for the container. If left empty, this value will not be specified by the client and defaulted by the server."))
   194  	cmd.Flags().BoolVarP(&o.Interactive, "stdin", "i", o.Interactive, i18n.T("Keep stdin open on the container(s) in the pod, even if nothing is attached."))
   195  	cmd.Flags().BoolVarP(&o.Quiet, "quiet", "q", o.Quiet, i18n.T("If true, suppress informational messages."))
   196  	cmd.Flags().BoolVar(&o.SameNode, "same-node", o.SameNode, i18n.T("When used with '--copy-to', schedule the copy of target Pod on the same node."))
   197  	cmd.Flags().BoolVar(&o.ShareProcesses, "share-processes", o.ShareProcesses, i18n.T("When used with '--copy-to', enable process namespace sharing in the copy."))
   198  	cmd.Flags().StringVar(&o.TargetContainer, "target", "", i18n.T("When using an ephemeral container, target processes in this container name."))
   199  	cmd.Flags().BoolVarP(&o.TTY, "tty", "t", o.TTY, i18n.T("Allocate a TTY for the debugging container."))
   200  	cmd.Flags().StringVar(&o.Profile, "profile", ProfileLegacy, i18n.T(`Options are "legacy", "general", "baseline", "netadmin", "restricted" or "sysadmin".`))
   201  	if cmdutil.DebugCustomProfile.IsEnabled() {
   202  		cmd.Flags().StringVar(&o.CustomProfileFile, "custom", o.CustomProfileFile, i18n.T("Path to a JSON file containing a partial container spec to customize built-in debug profiles."))
   203  	}
   204  }
   205  
   206  // Complete finishes run-time initialization of debug.DebugOptions.
   207  func (o *DebugOptions) Complete(restClientGetter genericclioptions.RESTClientGetter, cmd *cobra.Command, args []string) error {
   208  	var err error
   209  
   210  	o.PullPolicy = corev1.PullPolicy(cmdutil.GetFlagString(cmd, "image-pull-policy"))
   211  
   212  	// Arguments
   213  	argsLen := cmd.ArgsLenAtDash()
   214  	o.TargetNames = args
   215  	// If there is a dash and there are args after the dash, extract the args.
   216  	if argsLen >= 0 && len(args) > argsLen {
   217  		o.TargetNames, o.Args = args[:argsLen], args[argsLen:]
   218  	}
   219  
   220  	// Attach
   221  	attachFlag := cmd.Flags().Lookup("attach")
   222  	if !attachFlag.Changed && o.Interactive {
   223  		o.Attach = true
   224  	}
   225  
   226  	// Downstream tools may want to use their own customized
   227  	// attach function to do extra work or use attach command
   228  	// with different flags instead of the static one defined in
   229  	// handleAttachPod. But if this function is not set explicitly,
   230  	// we fall back to default.
   231  	if o.AttachFunc == nil {
   232  		o.AttachFunc = o.handleAttachPod
   233  	}
   234  
   235  	// Environment
   236  	envStrings, err := cmd.Flags().GetStringToString("env")
   237  	if err != nil {
   238  		return fmt.Errorf("internal error getting env flag: %v", err)
   239  	}
   240  	for k, v := range envStrings {
   241  		o.Env = append(o.Env, corev1.EnvVar{Name: k, Value: v})
   242  	}
   243  
   244  	// Namespace
   245  	o.Namespace, o.explicitNamespace, err = restClientGetter.ToRawKubeConfigLoader().Namespace()
   246  	if err != nil {
   247  		return err
   248  	}
   249  
   250  	// Record flags that the user explicitly changed from their defaults
   251  	o.attachChanged = cmd.Flags().Changed("attach")
   252  	o.shareProcessedChanged = cmd.Flags().Changed("share-processes")
   253  
   254  	// Set default WarningPrinter
   255  	if o.WarningPrinter == nil {
   256  		o.WarningPrinter = printers.NewWarningPrinter(o.ErrOut, printers.WarningPrinterOptions{Color: term.AllowsColorOutput(o.ErrOut)})
   257  	}
   258  
   259  	if o.Applier == nil {
   260  		applier, err := NewProfileApplier(o.Profile)
   261  		if err != nil {
   262  			return err
   263  		}
   264  		o.Applier = applier
   265  	}
   266  
   267  	if o.CustomProfileFile != "" {
   268  		customProfileBytes, err := os.ReadFile(o.CustomProfileFile)
   269  		if err != nil {
   270  			return fmt.Errorf("must pass a container spec json file for custom profile: %w", err)
   271  		}
   272  
   273  		err = json.Unmarshal(customProfileBytes, &o.CustomProfile)
   274  		if err != nil {
   275  			return fmt.Errorf("%s does not contain a valid container spec: %w", o.CustomProfileFile, err)
   276  		}
   277  	}
   278  
   279  	clientConfig, err := restClientGetter.ToRESTConfig()
   280  	if err != nil {
   281  		return err
   282  	}
   283  
   284  	client, err := kubernetes.NewForConfig(clientConfig)
   285  	if err != nil {
   286  		return err
   287  	}
   288  
   289  	o.podClient = client.CoreV1()
   290  
   291  	o.Builder = resource.NewBuilder(restClientGetter)
   292  
   293  	return nil
   294  }
   295  
   296  // Validate checks that the provided debug options are specified.
   297  func (o *DebugOptions) Validate() error {
   298  	// Attach
   299  	if o.Attach && o.attachChanged && len(o.Image) == 0 && len(o.Container) == 0 {
   300  		return fmt.Errorf("you must specify --container or create a new container using --image in order to attach.")
   301  	}
   302  
   303  	// CopyTo
   304  	if len(o.CopyTo) > 0 {
   305  		if len(o.Image) == 0 && len(o.SetImages) == 0 && len(o.Args) == 0 {
   306  			return fmt.Errorf("you must specify --image, --set-image or command arguments.")
   307  		}
   308  		if len(o.Args) > 0 && len(o.Container) == 0 && len(o.Image) == 0 {
   309  			return fmt.Errorf("you must specify an existing container or a new image when specifying args.")
   310  		}
   311  	} else {
   312  		// These flags are exclusive to --copy-to
   313  		switch {
   314  		case o.Replace:
   315  			return fmt.Errorf("--replace may only be used with --copy-to.")
   316  		case o.SameNode:
   317  			return fmt.Errorf("--same-node may only be used with --copy-to.")
   318  		case len(o.SetImages) > 0:
   319  			return fmt.Errorf("--set-image may only be used with --copy-to.")
   320  		case len(o.Image) == 0:
   321  			return fmt.Errorf("you must specify --image when not using --copy-to.")
   322  		}
   323  	}
   324  
   325  	// Image
   326  	if len(o.Image) > 0 && !reference.ReferenceRegexp.MatchString(o.Image) {
   327  		return fmt.Errorf("invalid image name %q: %v", o.Image, reference.ErrReferenceInvalidFormat)
   328  	}
   329  
   330  	// Name
   331  	if len(o.TargetNames) == 0 && len(o.FilenameOptions.Filenames) == 0 {
   332  		return fmt.Errorf("NAME or filename is required for debug")
   333  	}
   334  
   335  	// Pull Policy
   336  	switch o.PullPolicy {
   337  	case corev1.PullAlways, corev1.PullIfNotPresent, corev1.PullNever, "":
   338  		// continue
   339  	default:
   340  		return fmt.Errorf("invalid image pull policy: %s", o.PullPolicy)
   341  	}
   342  
   343  	// SetImages
   344  	for name, image := range o.SetImages {
   345  		if !reference.ReferenceRegexp.MatchString(image) {
   346  			return fmt.Errorf("invalid image name %q for container %q: %v", image, name, reference.ErrReferenceInvalidFormat)
   347  		}
   348  	}
   349  
   350  	// TargetContainer
   351  	if len(o.TargetContainer) > 0 {
   352  		if len(o.CopyTo) > 0 {
   353  			return fmt.Errorf("--target is incompatible with --copy-to. Use --share-processes instead.")
   354  		}
   355  		if !o.Quiet {
   356  			fmt.Fprintf(o.Out, "Targeting container %q. If you don't see processes from this container it may be because the container runtime doesn't support this feature.\n", o.TargetContainer)
   357  			// TODO(verb): Add a list of supported container runtimes to https://kubernetes.io/docs/concepts/workloads/pods/ephemeral-containers/ and then link here.
   358  		}
   359  	}
   360  
   361  	// TTY
   362  	if o.TTY && !o.Interactive {
   363  		return fmt.Errorf("-i/--stdin is required for containers with -t/--tty=true")
   364  	}
   365  
   366  	// WarningPrinter
   367  	if o.WarningPrinter == nil {
   368  		return fmt.Errorf("WarningPrinter can not be used without initialization")
   369  	}
   370  
   371  	if o.CustomProfile != nil {
   372  		if o.CustomProfile.Name != "" || len(o.CustomProfile.Command) > 0 || o.CustomProfile.Image != "" || o.CustomProfile.Lifecycle != nil || len(o.CustomProfile.VolumeDevices) > 0 {
   373  			return fmt.Errorf("name, command, image, lifecycle and volume devices are not modifiable via custom profile")
   374  		}
   375  	}
   376  
   377  	return nil
   378  }
   379  
   380  // Run executes a kubectl debug.
   381  func (o *DebugOptions) Run(restClientGetter genericclioptions.RESTClientGetter, cmd *cobra.Command) error {
   382  	ctx := context.Background()
   383  
   384  	r := o.Builder.
   385  		WithScheme(scheme.Scheme, scheme.Scheme.PrioritizedVersionsAllGroups()...).
   386  		FilenameParam(o.explicitNamespace, &o.FilenameOptions).
   387  		NamespaceParam(o.Namespace).DefaultNamespace().ResourceNames("pods", o.TargetNames...).
   388  		Do()
   389  	if err := r.Err(); err != nil {
   390  		return err
   391  	}
   392  
   393  	err := r.Visit(func(info *resource.Info, err error) error {
   394  		if err != nil {
   395  			// TODO(verb): configurable early return
   396  			return err
   397  		}
   398  
   399  		var (
   400  			debugPod      *corev1.Pod
   401  			containerName string
   402  			visitErr      error
   403  		)
   404  		switch obj := info.Object.(type) {
   405  		case *corev1.Node:
   406  			debugPod, containerName, visitErr = o.visitNode(ctx, obj)
   407  		case *corev1.Pod:
   408  			debugPod, containerName, visitErr = o.visitPod(ctx, obj)
   409  		default:
   410  			visitErr = fmt.Errorf("%q not supported by debug", info.Mapping.GroupVersionKind)
   411  		}
   412  		if visitErr != nil {
   413  			return visitErr
   414  		}
   415  
   416  		if o.Attach && len(containerName) > 0 && o.AttachFunc != nil {
   417  			if err := o.AttachFunc(ctx, restClientGetter, cmd.Parent().CommandPath(), debugPod.Namespace, debugPod.Name, containerName); err != nil {
   418  				return err
   419  			}
   420  		}
   421  
   422  		return nil
   423  	})
   424  
   425  	return err
   426  }
   427  
   428  // visitNode handles debugging for node targets by creating a privileged pod running in the host namespaces.
   429  // Returns an already created pod and container name for subsequent attach, if applicable.
   430  func (o *DebugOptions) visitNode(ctx context.Context, node *corev1.Node) (*corev1.Pod, string, error) {
   431  	pods := o.podClient.Pods(o.Namespace)
   432  	debugPod, err := o.generateNodeDebugPod(node)
   433  	if err != nil {
   434  		return nil, "", err
   435  	}
   436  	newPod, err := pods.Create(ctx, debugPod, metav1.CreateOptions{})
   437  	if err != nil {
   438  		return nil, "", err
   439  	}
   440  
   441  	return newPod, newPod.Spec.Containers[0].Name, nil
   442  }
   443  
   444  // visitPod handles debugging for pod targets by (depending on options):
   445  //  1. Creating an ephemeral debug container in an existing pod, OR
   446  //  2. Making a copy of pod with certain attributes changed
   447  //
   448  // visitPod returns a pod and debug container name for subsequent attach, if applicable.
   449  func (o *DebugOptions) visitPod(ctx context.Context, pod *corev1.Pod) (*corev1.Pod, string, error) {
   450  	if len(o.CopyTo) > 0 {
   451  		return o.debugByCopy(ctx, pod)
   452  	}
   453  	return o.debugByEphemeralContainer(ctx, pod)
   454  }
   455  
   456  // debugByEphemeralContainer runs an EphemeralContainer in the target Pod for use as a debug container
   457  func (o *DebugOptions) debugByEphemeralContainer(ctx context.Context, pod *corev1.Pod) (*corev1.Pod, string, error) {
   458  	klog.V(2).Infof("existing ephemeral containers: %v", pod.Spec.EphemeralContainers)
   459  	podJS, err := json.Marshal(pod)
   460  	if err != nil {
   461  		return nil, "", fmt.Errorf("error creating JSON for pod: %v", err)
   462  	}
   463  
   464  	debugPod, debugContainer, err := o.generateDebugContainer(pod)
   465  	if err != nil {
   466  		return nil, "", err
   467  	}
   468  	klog.V(2).Infof("new ephemeral container: %#v", debugContainer)
   469  
   470  	debugJS, err := json.Marshal(debugPod)
   471  	if err != nil {
   472  		return nil, "", fmt.Errorf("error creating JSON for debug container: %v", err)
   473  	}
   474  
   475  	patch, err := strategicpatch.CreateTwoWayMergePatch(podJS, debugJS, pod)
   476  	if err != nil {
   477  		return nil, "", fmt.Errorf("error creating patch to add debug container: %v", err)
   478  	}
   479  	klog.V(2).Infof("generated strategic merge patch for debug container: %s", patch)
   480  
   481  	pods := o.podClient.Pods(pod.Namespace)
   482  	result, err := pods.Patch(ctx, pod.Name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}, "ephemeralcontainers")
   483  	if err != nil {
   484  		// The apiserver will return a 404 when the EphemeralContainers feature is disabled because the `/ephemeralcontainers` subresource
   485  		// is missing. Unlike the 404 returned by a missing pod, the status details will be empty.
   486  		if serr, ok := err.(*errors.StatusError); ok && serr.Status().Reason == metav1.StatusReasonNotFound && serr.ErrStatus.Details.Name == "" {
   487  			return nil, "", fmt.Errorf("ephemeral containers are disabled for this cluster (error from server: %q)", err)
   488  		}
   489  
   490  		return nil, "", err
   491  	}
   492  
   493  	return result, debugContainer.Name, nil
   494  }
   495  
   496  // applyCustomProfile applies given partial container json file on to the profile
   497  // incorporated debug pod.
   498  func (o *DebugOptions) applyCustomProfile(debugPod *corev1.Pod, containerName string) error {
   499  	o.CustomProfile.Name = containerName
   500  	customJS, err := json.Marshal(o.CustomProfile)
   501  	if err != nil {
   502  		return fmt.Errorf("unable to marshall custom profile: %w", err)
   503  	}
   504  
   505  	var index int
   506  	found := false
   507  	for i, val := range debugPod.Spec.Containers {
   508  		if val.Name == containerName {
   509  			index = i
   510  			found = true
   511  			break
   512  		}
   513  	}
   514  
   515  	if !found {
   516  		return fmt.Errorf("unable to find the %s container in the pod %s", containerName, debugPod.Name)
   517  	}
   518  
   519  	var debugContainerJS []byte
   520  	debugContainerJS, err = json.Marshal(debugPod.Spec.Containers[index])
   521  	if err != nil {
   522  		return fmt.Errorf("unable to marshall container: %w", err)
   523  	}
   524  
   525  	patchedContainer, err := strategicpatch.StrategicMergePatch(debugContainerJS, customJS, corev1.Container{})
   526  	if err != nil {
   527  		return fmt.Errorf("error creating three way patch to add debug container: %w", err)
   528  	}
   529  
   530  	err = json.Unmarshal(patchedContainer, &debugPod.Spec.Containers[index])
   531  	if err != nil {
   532  		return fmt.Errorf("unable to unmarshall patched container to container: %w", err)
   533  	}
   534  
   535  	return nil
   536  }
   537  
   538  // applyCustomProfileEphemeral applies given partial container json file on to the profile
   539  // incorporated ephemeral container of the pod.
   540  func (o *DebugOptions) applyCustomProfileEphemeral(debugPod *corev1.Pod, containerName string) error {
   541  	o.CustomProfile.Name = containerName
   542  	customJS, err := json.Marshal(o.CustomProfile)
   543  	if err != nil {
   544  		return fmt.Errorf("unable to marshall custom profile: %w", err)
   545  	}
   546  
   547  	var index int
   548  	found := false
   549  	for i, val := range debugPod.Spec.EphemeralContainers {
   550  		if val.Name == containerName {
   551  			index = i
   552  			found = true
   553  			break
   554  		}
   555  	}
   556  
   557  	if !found {
   558  		return fmt.Errorf("unable to find the %s ephemeral container in the pod %s", containerName, debugPod.Name)
   559  	}
   560  
   561  	var debugContainerJS []byte
   562  	debugContainerJS, err = json.Marshal(debugPod.Spec.EphemeralContainers[index])
   563  	if err != nil {
   564  		return fmt.Errorf("unable to marshall ephemeral container:%w", err)
   565  	}
   566  
   567  	patchedContainer, err := strategicpatch.StrategicMergePatch(debugContainerJS, customJS, corev1.Container{})
   568  	if err != nil {
   569  		return fmt.Errorf("error creating three way patch to add debug container: %w", err)
   570  	}
   571  
   572  	err = json.Unmarshal(patchedContainer, &debugPod.Spec.EphemeralContainers[index])
   573  	if err != nil {
   574  		return fmt.Errorf("unable to unmarshall patched container to ephemeral container: %w", err)
   575  	}
   576  
   577  	return nil
   578  }
   579  
   580  // debugByCopy runs a copy of the target Pod with a debug container added or an original container modified
   581  func (o *DebugOptions) debugByCopy(ctx context.Context, pod *corev1.Pod) (*corev1.Pod, string, error) {
   582  	copied, dc, err := o.generatePodCopyWithDebugContainer(pod)
   583  	if err != nil {
   584  		return nil, "", err
   585  	}
   586  	created, err := o.podClient.Pods(copied.Namespace).Create(ctx, copied, metav1.CreateOptions{})
   587  	if err != nil {
   588  		return nil, "", err
   589  	}
   590  	if o.Replace {
   591  		err := o.podClient.Pods(pod.Namespace).Delete(ctx, pod.Name, *metav1.NewDeleteOptions(0))
   592  		if err != nil {
   593  			return nil, "", err
   594  		}
   595  	}
   596  	return created, dc, nil
   597  }
   598  
   599  // generateDebugContainer returns a debugging pod and an EphemeralContainer suitable for use as a debug container
   600  // in the given pod.
   601  func (o *DebugOptions) generateDebugContainer(pod *corev1.Pod) (*corev1.Pod, *corev1.EphemeralContainer, error) {
   602  	name := o.computeDebugContainerName(pod)
   603  	ec := &corev1.EphemeralContainer{
   604  		EphemeralContainerCommon: corev1.EphemeralContainerCommon{
   605  			Name:                     name,
   606  			Env:                      o.Env,
   607  			Image:                    o.Image,
   608  			ImagePullPolicy:          o.PullPolicy,
   609  			Stdin:                    o.Interactive,
   610  			TerminationMessagePolicy: corev1.TerminationMessageReadFile,
   611  			TTY:                      o.TTY,
   612  		},
   613  		TargetContainerName: o.TargetContainer,
   614  	}
   615  
   616  	if o.ArgsOnly {
   617  		ec.Args = o.Args
   618  	} else {
   619  		ec.Command = o.Args
   620  	}
   621  
   622  	copied := pod.DeepCopy()
   623  	copied.Spec.EphemeralContainers = append(copied.Spec.EphemeralContainers, *ec)
   624  	if err := o.Applier.Apply(copied, name, copied); err != nil {
   625  		return nil, nil, err
   626  	}
   627  
   628  	if o.CustomProfile != nil {
   629  		err := o.applyCustomProfileEphemeral(copied, ec.Name)
   630  		if err != nil {
   631  			return nil, nil, err
   632  		}
   633  	}
   634  
   635  	ec = &copied.Spec.EphemeralContainers[len(copied.Spec.EphemeralContainers)-1]
   636  
   637  	return copied, ec, nil
   638  }
   639  
   640  // generateNodeDebugPod generates a debugging pod that schedules on the specified node.
   641  // The generated pod will run in the host PID, Network & IPC namespaces, and it will have the node's filesystem mounted at /host.
   642  func (o *DebugOptions) generateNodeDebugPod(node *corev1.Node) (*corev1.Pod, error) {
   643  	cn := "debugger"
   644  	// Setting a user-specified container name doesn't make much difference when there's only one container,
   645  	// but the argument exists for pod debugging so it might be confusing if it didn't work here.
   646  	if len(o.Container) > 0 {
   647  		cn = o.Container
   648  	}
   649  
   650  	// The name of the debugging pod is based on the target node, and it's not configurable to
   651  	// limit the number of command line flags. There may be a collision on the name, but this
   652  	// should be rare enough that it's not worth the API round trip to check.
   653  	pn := fmt.Sprintf("node-debugger-%s-%s", node.Name, nameSuffixFunc(5))
   654  	if !o.Quiet {
   655  		fmt.Fprintf(o.Out, "Creating debugging pod %s with container %s on node %s.\n", pn, cn, node.Name)
   656  	}
   657  
   658  	p := &corev1.Pod{
   659  		ObjectMeta: metav1.ObjectMeta{
   660  			Name: pn,
   661  		},
   662  		Spec: corev1.PodSpec{
   663  			Containers: []corev1.Container{
   664  				{
   665  					Name:                     cn,
   666  					Env:                      o.Env,
   667  					Image:                    o.Image,
   668  					ImagePullPolicy:          o.PullPolicy,
   669  					Stdin:                    o.Interactive,
   670  					TerminationMessagePolicy: corev1.TerminationMessageReadFile,
   671  					TTY:                      o.TTY,
   672  				},
   673  			},
   674  			NodeName:      node.Name,
   675  			RestartPolicy: corev1.RestartPolicyNever,
   676  			Tolerations: []corev1.Toleration{
   677  				{
   678  					Operator: corev1.TolerationOpExists,
   679  				},
   680  			},
   681  		},
   682  	}
   683  
   684  	if o.ArgsOnly {
   685  		p.Spec.Containers[0].Args = o.Args
   686  	} else {
   687  		p.Spec.Containers[0].Command = o.Args
   688  	}
   689  
   690  	if err := o.Applier.Apply(p, cn, node); err != nil {
   691  		return nil, err
   692  	}
   693  
   694  	if o.CustomProfile != nil {
   695  		err := o.applyCustomProfile(p, cn)
   696  		if err != nil {
   697  			return nil, err
   698  		}
   699  	}
   700  
   701  	return p, nil
   702  }
   703  
   704  // generatePodCopyWithDebugContainer takes a Pod and returns a copy and the debug container name of that copy
   705  func (o *DebugOptions) generatePodCopyWithDebugContainer(pod *corev1.Pod) (*corev1.Pod, string, error) {
   706  	copied := &corev1.Pod{
   707  		ObjectMeta: metav1.ObjectMeta{
   708  			Name:        o.CopyTo,
   709  			Namespace:   pod.Namespace,
   710  			Annotations: pod.Annotations,
   711  		},
   712  		Spec: *pod.Spec.DeepCopy(),
   713  	}
   714  	// set EphemeralContainers to nil so that the copy of pod can be created
   715  	copied.Spec.EphemeralContainers = nil
   716  	// change ShareProcessNamespace configuration only when commanded explicitly
   717  	if o.shareProcessedChanged {
   718  		copied.Spec.ShareProcessNamespace = pointer.Bool(o.ShareProcesses)
   719  	}
   720  	if !o.SameNode {
   721  		copied.Spec.NodeName = ""
   722  	}
   723  
   724  	// Apply image mutations
   725  	for i, c := range copied.Spec.Containers {
   726  		override := o.SetImages["*"]
   727  		if img, ok := o.SetImages[c.Name]; ok {
   728  			override = img
   729  		}
   730  		if len(override) > 0 {
   731  			copied.Spec.Containers[i].Image = override
   732  		}
   733  	}
   734  
   735  	name, containerByName := o.Container, containerNameToRef(copied)
   736  
   737  	c, ok := containerByName[name]
   738  	if !ok {
   739  		// Adding a new debug container
   740  		if len(o.Image) == 0 {
   741  			if len(o.SetImages) > 0 {
   742  				// This was a --set-image only invocation
   743  				return copied, "", nil
   744  			}
   745  			return nil, "", fmt.Errorf("you must specify image when creating new container")
   746  		}
   747  
   748  		if len(name) == 0 {
   749  			name = o.computeDebugContainerName(copied)
   750  		}
   751  		copied.Spec.Containers = append(copied.Spec.Containers, corev1.Container{
   752  			Name:                     name,
   753  			TerminationMessagePolicy: corev1.TerminationMessageReadFile,
   754  		})
   755  		c = &copied.Spec.Containers[len(copied.Spec.Containers)-1]
   756  	}
   757  
   758  	if len(o.Args) > 0 {
   759  		if o.ArgsOnly {
   760  			c.Args = o.Args
   761  		} else {
   762  			c.Command = o.Args
   763  			c.Args = nil
   764  		}
   765  	}
   766  	if len(o.Env) > 0 {
   767  		c.Env = o.Env
   768  	}
   769  	if len(o.Image) > 0 {
   770  		c.Image = o.Image
   771  	}
   772  	if len(o.PullPolicy) > 0 {
   773  		c.ImagePullPolicy = o.PullPolicy
   774  	}
   775  	c.Stdin = o.Interactive
   776  	c.TTY = o.TTY
   777  
   778  	err := o.Applier.Apply(copied, c.Name, pod)
   779  	if err != nil {
   780  		return nil, "", err
   781  	}
   782  
   783  	if o.CustomProfile != nil {
   784  		err = o.applyCustomProfile(copied, name)
   785  		if err != nil {
   786  			return nil, "", err
   787  		}
   788  	}
   789  
   790  	return copied, name, nil
   791  }
   792  
   793  func (o *DebugOptions) computeDebugContainerName(pod *corev1.Pod) string {
   794  	if len(o.Container) > 0 {
   795  		return o.Container
   796  	}
   797  
   798  	cn, containerByName := "", containerNameToRef(pod)
   799  	for len(cn) == 0 || (containerByName[cn] != nil) {
   800  		cn = fmt.Sprintf("debugger-%s", nameSuffixFunc(5))
   801  	}
   802  	if !o.Quiet {
   803  		fmt.Fprintf(o.Out, "Defaulting debug container name to %s.\n", cn)
   804  	}
   805  	return cn
   806  }
   807  
   808  func containerNameToRef(pod *corev1.Pod) map[string]*corev1.Container {
   809  	names := map[string]*corev1.Container{}
   810  	for i := range pod.Spec.Containers {
   811  		ref := &pod.Spec.Containers[i]
   812  		names[ref.Name] = ref
   813  	}
   814  	for i := range pod.Spec.InitContainers {
   815  		ref := &pod.Spec.InitContainers[i]
   816  		names[ref.Name] = ref
   817  	}
   818  	for i := range pod.Spec.EphemeralContainers {
   819  		ref := (*corev1.Container)(&pod.Spec.EphemeralContainers[i].EphemeralContainerCommon)
   820  		names[ref.Name] = ref
   821  	}
   822  	return names
   823  }
   824  
   825  // waitForContainer watches the given pod until the container is running
   826  func (o *DebugOptions) waitForContainer(ctx context.Context, ns, podName, containerName string) (*corev1.Pod, error) {
   827  	// TODO: expose the timeout
   828  	ctx, cancel := watchtools.ContextWithOptionalTimeout(ctx, 0*time.Second)
   829  	defer cancel()
   830  
   831  	fieldSelector := fields.OneTermEqualSelector("metadata.name", podName).String()
   832  	lw := &cache.ListWatch{
   833  		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
   834  			options.FieldSelector = fieldSelector
   835  			return o.podClient.Pods(ns).List(ctx, options)
   836  		},
   837  		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
   838  			options.FieldSelector = fieldSelector
   839  			return o.podClient.Pods(ns).Watch(ctx, options)
   840  		},
   841  	}
   842  
   843  	intr := interrupt.New(nil, cancel)
   844  	var result *corev1.Pod
   845  	err := intr.Run(func() error {
   846  		ev, err := watchtools.UntilWithSync(ctx, lw, &corev1.Pod{}, nil, func(ev watch.Event) (bool, error) {
   847  			klog.V(2).Infof("watch received event %q with object %T", ev.Type, ev.Object)
   848  			switch ev.Type {
   849  			case watch.Deleted:
   850  				return false, errors.NewNotFound(schema.GroupResource{Resource: "pods"}, "")
   851  			}
   852  
   853  			p, ok := ev.Object.(*corev1.Pod)
   854  			if !ok {
   855  				return false, fmt.Errorf("watch did not return a pod: %v", ev.Object)
   856  			}
   857  
   858  			s := getContainerStatusByName(p, containerName)
   859  			if s == nil {
   860  				return false, nil
   861  			}
   862  			klog.V(2).Infof("debug container status is %v", s)
   863  			if s.State.Running != nil || s.State.Terminated != nil {
   864  				return true, nil
   865  			}
   866  			if !o.Quiet && s.State.Waiting != nil && s.State.Waiting.Message != "" {
   867  				o.WarningPrinter.Print(fmt.Sprintf("container %s: %s", containerName, s.State.Waiting.Message))
   868  			}
   869  			return false, nil
   870  		})
   871  		if ev != nil {
   872  			result = ev.Object.(*corev1.Pod)
   873  		}
   874  		return err
   875  	})
   876  
   877  	return result, err
   878  }
   879  
   880  func (o *DebugOptions) handleAttachPod(ctx context.Context, restClientGetter genericclioptions.RESTClientGetter, cmdPath string, ns, podName, containerName string) error {
   881  	opts := &attach.AttachOptions{
   882  		StreamOptions: exec.StreamOptions{
   883  			IOStreams: o.IOStreams,
   884  			Stdin:     o.Interactive,
   885  			TTY:       o.TTY,
   886  			Quiet:     o.Quiet,
   887  		},
   888  		CommandName: cmdPath + " attach",
   889  
   890  		Attach: &attach.DefaultRemoteAttach{},
   891  	}
   892  	config, err := restClientGetter.ToRESTConfig()
   893  	if err != nil {
   894  		return err
   895  	}
   896  	opts.Config = config
   897  	opts.AttachFunc = attach.DefaultAttachFunc
   898  
   899  	pod, err := o.waitForContainer(ctx, ns, podName, containerName)
   900  	if err != nil {
   901  		return err
   902  	}
   903  
   904  	opts.Namespace = ns
   905  	opts.Pod = pod
   906  	opts.PodName = podName
   907  	opts.ContainerName = containerName
   908  	if opts.AttachFunc == nil {
   909  		opts.AttachFunc = attach.DefaultAttachFunc
   910  	}
   911  
   912  	status := getContainerStatusByName(pod, containerName)
   913  	if status == nil {
   914  		// impossible path
   915  		return fmt.Errorf("error getting container status of container name %q: %+v", containerName, err)
   916  	}
   917  	if status.State.Terminated != nil {
   918  		klog.V(1).Info("Ephemeral container terminated, falling back to logs")
   919  		return logOpts(restClientGetter, pod, opts)
   920  	}
   921  
   922  	if err := opts.Run(); err != nil {
   923  		fmt.Fprintf(opts.ErrOut, "warning: couldn't attach to pod/%s, falling back to streaming logs: %v\n", podName, err)
   924  		return logOpts(restClientGetter, pod, opts)
   925  	}
   926  	return nil
   927  }
   928  
   929  func getContainerStatusByName(pod *corev1.Pod, containerName string) *corev1.ContainerStatus {
   930  	allContainerStatus := [][]corev1.ContainerStatus{pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses, pod.Status.EphemeralContainerStatuses}
   931  	for _, statusSlice := range allContainerStatus {
   932  		for i := range statusSlice {
   933  			if statusSlice[i].Name == containerName {
   934  				return &statusSlice[i]
   935  			}
   936  		}
   937  	}
   938  	return nil
   939  }
   940  
   941  // logOpts logs output from opts to the pods log.
   942  func logOpts(restClientGetter genericclioptions.RESTClientGetter, pod *corev1.Pod, opts *attach.AttachOptions) error {
   943  	ctrName, err := opts.GetContainerName(pod)
   944  	if err != nil {
   945  		return err
   946  	}
   947  
   948  	requests, err := polymorphichelpers.LogsForObjectFn(restClientGetter, pod, &corev1.PodLogOptions{Container: ctrName}, opts.GetPodTimeout, false)
   949  	if err != nil {
   950  		return err
   951  	}
   952  	for _, request := range requests {
   953  		if err := logs.DefaultConsumeRequest(request, opts.Out); err != nil {
   954  			return err
   955  		}
   956  	}
   957  
   958  	return nil
   959  }
   960
View as plain text