process.go

Documentation: github.com/Microsoft/hcsshim/internal/hcs

     1  //go:build windows
     2  
     3  package hcs
     4  
     5  import (
     6  	"context"
     7  	"encoding/json"
     8  	"errors"
     9  	"io"
    10  	"os"
    11  	"sync"
    12  	"syscall"
    13  	"time"
    14  
    15  	"github.com/Microsoft/hcsshim/internal/cow"
    16  	"github.com/Microsoft/hcsshim/internal/log"
    17  	"github.com/Microsoft/hcsshim/internal/oc"
    18  	"github.com/Microsoft/hcsshim/internal/vmcompute"
    19  	"go.opencensus.io/trace"
    20  )
    21  
    22  // ContainerError is an error encountered in HCS
    23  type Process struct {
    24  	handleLock          sync.RWMutex
    25  	handle              vmcompute.HcsProcess
    26  	processID           int
    27  	system              *System
    28  	hasCachedStdio      bool
    29  	stdioLock           sync.Mutex
    30  	stdin               io.WriteCloser
    31  	stdout              io.ReadCloser
    32  	stderr              io.ReadCloser
    33  	callbackNumber      uintptr
    34  	killSignalDelivered bool
    35  
    36  	closedWaitOnce sync.Once
    37  	waitBlock      chan struct{}
    38  	exitCode       int
    39  	waitError      error
    40  }
    41  
    42  var _ cow.Process = &Process{}
    43  
    44  func newProcess(process vmcompute.HcsProcess, processID int, computeSystem *System) *Process {
    45  	return &Process{
    46  		handle:    process,
    47  		processID: processID,
    48  		system:    computeSystem,
    49  		waitBlock: make(chan struct{}),
    50  	}
    51  }
    52  
    53  type processModifyRequest struct {
    54  	Operation   string
    55  	ConsoleSize *consoleSize `json:",omitempty"`
    56  	CloseHandle *closeHandle `json:",omitempty"`
    57  }
    58  
    59  type consoleSize struct {
    60  	Height uint16
    61  	Width  uint16
    62  }
    63  
    64  type closeHandle struct {
    65  	Handle string
    66  }
    67  
    68  type processStatus struct {
    69  	ProcessID      uint32
    70  	Exited         bool
    71  	ExitCode       uint32
    72  	LastWaitResult int32
    73  }
    74  
    75  const stdIn string = "StdIn"
    76  
    77  const (
    78  	modifyConsoleSize string = "ConsoleSize"
    79  	modifyCloseHandle string = "CloseHandle"
    80  )
    81  
    82  // Pid returns the process ID of the process within the container.
    83  func (process *Process) Pid() int {
    84  	return process.processID
    85  }
    86  
    87  // SystemID returns the ID of the process's compute system.
    88  func (process *Process) SystemID() string {
    89  	return process.system.ID()
    90  }
    91  
    92  func (process *Process) processSignalResult(ctx context.Context, err error) (bool, error) {
    93  	switch err {
    94  	case nil:
    95  		return true, nil
    96  	case ErrVmcomputeOperationInvalidState, ErrComputeSystemDoesNotExist, ErrElementNotFound:
    97  		if !process.stopped() {
    98  			// The process should be gone, but we have not received the notification.
    99  			// After a second, force unblock the process wait to work around a possible
   100  			// deadlock in the HCS.
   101  			go func() {
   102  				time.Sleep(time.Second)
   103  				process.closedWaitOnce.Do(func() {
   104  					log.G(ctx).WithError(err).Warn("force unblocking process waits")
   105  					process.exitCode = -1
   106  					process.waitError = err
   107  					close(process.waitBlock)
   108  				})
   109  			}()
   110  		}
   111  		return false, nil
   112  	default:
   113  		return false, err
   114  	}
   115  }
   116  
   117  // Signal signals the process with `options`.
   118  //
   119  // For LCOW `guestresource.SignalProcessOptionsLCOW`.
   120  //
   121  // For WCOW `guestresource.SignalProcessOptionsWCOW`.
   122  func (process *Process) Signal(ctx context.Context, options interface{}) (bool, error) {
   123  	process.handleLock.RLock()
   124  	defer process.handleLock.RUnlock()
   125  
   126  	operation := "hcs::Process::Signal"
   127  
   128  	if process.handle == 0 {
   129  		return false, makeProcessError(process, operation, ErrAlreadyClosed, nil)
   130  	}
   131  
   132  	optionsb, err := json.Marshal(options)
   133  	if err != nil {
   134  		return false, err
   135  	}
   136  
   137  	resultJSON, err := vmcompute.HcsSignalProcess(ctx, process.handle, string(optionsb))
   138  	events := processHcsResult(ctx, resultJSON)
   139  	delivered, err := process.processSignalResult(ctx, err)
   140  	if err != nil {
   141  		err = makeProcessError(process, operation, err, events)
   142  	}
   143  	return delivered, err
   144  }
   145  
   146  // Kill signals the process to terminate but does not wait for it to finish terminating.
   147  func (process *Process) Kill(ctx context.Context) (bool, error) {
   148  	process.handleLock.RLock()
   149  	defer process.handleLock.RUnlock()
   150  
   151  	operation := "hcs::Process::Kill"
   152  
   153  	if process.handle == 0 {
   154  		return false, makeProcessError(process, operation, ErrAlreadyClosed, nil)
   155  	}
   156  
   157  	if process.stopped() {
   158  		return false, makeProcessError(process, operation, ErrProcessAlreadyStopped, nil)
   159  	}
   160  
   161  	if process.killSignalDelivered {
   162  		// A kill signal has already been sent to this process. Sending a second
   163  		// one offers no real benefit, as processes cannot stop themselves from
   164  		// being terminated, once a TerminateProcess has been issued. Sending a
   165  		// second kill may result in a number of errors (two of which detailed bellow)
   166  		// and which we can avoid handling.
   167  		return true, nil
   168  	}
   169  
   170  	// HCS serializes the signals sent to a target pid per compute system handle.
   171  	// To avoid SIGKILL being serialized behind other signals, we open a new compute
   172  	// system handle to deliver the kill signal.
   173  	// If the calls to opening a new compute system handle fail, we forcefully
   174  	// terminate the container itself so that no container is left behind
   175  	hcsSystem, err := OpenComputeSystem(ctx, process.system.id)
   176  	if err != nil {
   177  		// log error and force termination of container
   178  		log.G(ctx).WithField("err", err).Error("OpenComputeSystem() call failed")
   179  		err = process.system.Terminate(ctx)
   180  		// if the Terminate() call itself ever failed, log and return error
   181  		if err != nil {
   182  			log.G(ctx).WithField("err", err).Error("Terminate() call failed")
   183  			return false, err
   184  		}
   185  		process.system.Close()
   186  		return true, nil
   187  	}
   188  	defer hcsSystem.Close()
   189  
   190  	newProcessHandle, err := hcsSystem.OpenProcess(ctx, process.Pid())
   191  	if err != nil {
   192  		// Return true only if the target process has either already
   193  		// exited, or does not exist.
   194  		if IsAlreadyStopped(err) {
   195  			return true, nil
   196  		} else {
   197  			return false, err
   198  		}
   199  	}
   200  	defer newProcessHandle.Close()
   201  
   202  	resultJSON, err := vmcompute.HcsTerminateProcess(ctx, newProcessHandle.handle)
   203  	if err != nil {
   204  		// We still need to check these two cases, as processes may still be killed by an
   205  		// external actor (human operator, OOM, random script etc).
   206  		if errors.Is(err, os.ErrPermission) || IsAlreadyStopped(err) {
   207  			// There are two cases where it should be safe to ignore an error returned
   208  			// by HcsTerminateProcess. The first one is cause by the fact that
   209  			// HcsTerminateProcess ends up calling TerminateProcess in the context
   210  			// of a container. According to the TerminateProcess documentation:
   211  			// https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-terminateprocess#remarks
   212  			// After a process has terminated, call to TerminateProcess with open
   213  			// handles to the process fails with ERROR_ACCESS_DENIED (5) error code.
   214  			// It's safe to ignore this error here. HCS should always have permissions
   215  			// to kill processes inside any container. So an ERROR_ACCESS_DENIED
   216  			// is unlikely to be anything else than what the ending remarks in the
   217  			// documentation states.
   218  			//
   219  			// The second case is generated by hcs itself, if for any reason HcsTerminateProcess
   220  			// is called twice in a very short amount of time. In such cases, hcs may return
   221  			// HCS_E_PROCESS_ALREADY_STOPPED.
   222  			return true, nil
   223  		}
   224  	}
   225  	events := processHcsResult(ctx, resultJSON)
   226  	delivered, err := newProcessHandle.processSignalResult(ctx, err)
   227  	if err != nil {
   228  		err = makeProcessError(newProcessHandle, operation, err, events)
   229  	}
   230  
   231  	process.killSignalDelivered = delivered
   232  	return delivered, err
   233  }
   234  
   235  // waitBackground waits for the process exit notification. Once received sets
   236  // `process.waitError` (if any) and unblocks all `Wait` calls.
   237  //
   238  // This MUST be called exactly once per `process.handle` but `Wait` is safe to
   239  // call multiple times.
   240  func (process *Process) waitBackground() {
   241  	operation := "hcs::Process::waitBackground"
   242  	ctx, span := oc.StartSpan(context.Background(), operation)
   243  	defer span.End()
   244  	span.AddAttributes(
   245  		trace.StringAttribute("cid", process.SystemID()),
   246  		trace.Int64Attribute("pid", int64(process.processID)))
   247  
   248  	var (
   249  		err            error
   250  		exitCode       = -1
   251  		propertiesJSON string
   252  		resultJSON     string
   253  	)
   254  
   255  	err = waitForNotification(ctx, process.callbackNumber, hcsNotificationProcessExited, nil)
   256  	if err != nil {
   257  		err = makeProcessError(process, operation, err, nil)
   258  		log.G(ctx).WithError(err).Error("failed wait")
   259  	} else {
   260  		process.handleLock.RLock()
   261  		defer process.handleLock.RUnlock()
   262  
   263  		// Make sure we didnt race with Close() here
   264  		if process.handle != 0 {
   265  			propertiesJSON, resultJSON, err = vmcompute.HcsGetProcessProperties(ctx, process.handle)
   266  			events := processHcsResult(ctx, resultJSON)
   267  			if err != nil {
   268  				err = makeProcessError(process, operation, err, events)
   269  			} else {
   270  				properties := &processStatus{}
   271  				err = json.Unmarshal([]byte(propertiesJSON), properties)
   272  				if err != nil {
   273  					err = makeProcessError(process, operation, err, nil)
   274  				} else {
   275  					if properties.LastWaitResult != 0 {
   276  						log.G(ctx).WithField("wait-result", properties.LastWaitResult).Warning("non-zero last wait result")
   277  					} else {
   278  						exitCode = int(properties.ExitCode)
   279  					}
   280  				}
   281  			}
   282  		}
   283  	}
   284  	log.G(ctx).WithField("exitCode", exitCode).Debug("process exited")
   285  
   286  	process.closedWaitOnce.Do(func() {
   287  		process.exitCode = exitCode
   288  		process.waitError = err
   289  		close(process.waitBlock)
   290  	})
   291  	oc.SetSpanStatus(span, err)
   292  }
   293  
   294  // Wait waits for the process to exit. If the process has already exited returns
   295  // the previous error (if any).
   296  func (process *Process) Wait() error {
   297  	<-process.waitBlock
   298  	return process.waitError
   299  }
   300  
   301  // Exited returns if the process has stopped
   302  func (process *Process) stopped() bool {
   303  	select {
   304  	case <-process.waitBlock:
   305  		return true
   306  	default:
   307  		return false
   308  	}
   309  }
   310  
   311  // ResizeConsole resizes the console of the process.
   312  func (process *Process) ResizeConsole(ctx context.Context, width, height uint16) error {
   313  	process.handleLock.RLock()
   314  	defer process.handleLock.RUnlock()
   315  
   316  	operation := "hcs::Process::ResizeConsole"
   317  
   318  	if process.handle == 0 {
   319  		return makeProcessError(process, operation, ErrAlreadyClosed, nil)
   320  	}
   321  
   322  	modifyRequest := processModifyRequest{
   323  		Operation: modifyConsoleSize,
   324  		ConsoleSize: &consoleSize{
   325  			Height: height,
   326  			Width:  width,
   327  		},
   328  	}
   329  
   330  	modifyRequestb, err := json.Marshal(modifyRequest)
   331  	if err != nil {
   332  		return err
   333  	}
   334  
   335  	resultJSON, err := vmcompute.HcsModifyProcess(ctx, process.handle, string(modifyRequestb))
   336  	events := processHcsResult(ctx, resultJSON)
   337  	if err != nil {
   338  		return makeProcessError(process, operation, err, events)
   339  	}
   340  
   341  	return nil
   342  }
   343  
   344  // ExitCode returns the exit code of the process. The process must have
   345  // already terminated.
   346  func (process *Process) ExitCode() (int, error) {
   347  	if !process.stopped() {
   348  		return -1, makeProcessError(process, "hcs::Process::ExitCode", ErrInvalidProcessState, nil)
   349  	}
   350  	if process.waitError != nil {
   351  		return -1, process.waitError
   352  	}
   353  	return process.exitCode, nil
   354  }
   355  
   356  // StdioLegacy returns the stdin, stdout, and stderr pipes, respectively. Closing
   357  // these pipes does not close the underlying pipes. Once returned, these pipes
   358  // are the responsibility of the caller to close.
   359  func (process *Process) StdioLegacy() (_ io.WriteCloser, _ io.ReadCloser, _ io.ReadCloser, err error) {
   360  	operation := "hcs::Process::StdioLegacy"
   361  	ctx, span := oc.StartSpan(context.Background(), operation)
   362  	defer span.End()
   363  	defer func() { oc.SetSpanStatus(span, err) }()
   364  	span.AddAttributes(
   365  		trace.StringAttribute("cid", process.SystemID()),
   366  		trace.Int64Attribute("pid", int64(process.processID)))
   367  
   368  	process.handleLock.RLock()
   369  	defer process.handleLock.RUnlock()
   370  
   371  	if process.handle == 0 {
   372  		return nil, nil, nil, makeProcessError(process, operation, ErrAlreadyClosed, nil)
   373  	}
   374  
   375  	process.stdioLock.Lock()
   376  	defer process.stdioLock.Unlock()
   377  	if process.hasCachedStdio {
   378  		stdin, stdout, stderr := process.stdin, process.stdout, process.stderr
   379  		process.stdin, process.stdout, process.stderr = nil, nil, nil
   380  		process.hasCachedStdio = false
   381  		return stdin, stdout, stderr, nil
   382  	}
   383  
   384  	processInfo, resultJSON, err := vmcompute.HcsGetProcessInfo(ctx, process.handle)
   385  	events := processHcsResult(ctx, resultJSON)
   386  	if err != nil {
   387  		return nil, nil, nil, makeProcessError(process, operation, err, events)
   388  	}
   389  
   390  	pipes, err := makeOpenFiles([]syscall.Handle{processInfo.StdInput, processInfo.StdOutput, processInfo.StdError})
   391  	if err != nil {
   392  		return nil, nil, nil, makeProcessError(process, operation, err, nil)
   393  	}
   394  
   395  	return pipes[0], pipes[1], pipes[2], nil
   396  }
   397  
   398  // Stdio returns the stdin, stdout, and stderr pipes, respectively.
   399  // To close them, close the process handle, or use the `CloseStd*` functions.
   400  func (process *Process) Stdio() (stdin io.Writer, stdout, stderr io.Reader) {
   401  	process.stdioLock.Lock()
   402  	defer process.stdioLock.Unlock()
   403  	return process.stdin, process.stdout, process.stderr
   404  }
   405  
   406  // CloseStdin closes the write side of the stdin pipe so that the process is
   407  // notified on the read side that there is no more data in stdin.
   408  func (process *Process) CloseStdin(ctx context.Context) (err error) {
   409  	operation := "hcs::Process::CloseStdin"
   410  	ctx, span := trace.StartSpan(ctx, operation)
   411  	defer span.End()
   412  	defer func() { oc.SetSpanStatus(span, err) }()
   413  	span.AddAttributes(
   414  		trace.StringAttribute("cid", process.SystemID()),
   415  		trace.Int64Attribute("pid", int64(process.processID)))
   416  
   417  	process.handleLock.RLock()
   418  	defer process.handleLock.RUnlock()
   419  
   420  	if process.handle == 0 {
   421  		return makeProcessError(process, operation, ErrAlreadyClosed, nil)
   422  	}
   423  
   424  	//HcsModifyProcess request to close stdin will fail if the process has already exited
   425  	if !process.stopped() {
   426  		modifyRequest := processModifyRequest{
   427  			Operation: modifyCloseHandle,
   428  			CloseHandle: &closeHandle{
   429  				Handle: stdIn,
   430  			},
   431  		}
   432  
   433  		modifyRequestb, err := json.Marshal(modifyRequest)
   434  		if err != nil {
   435  			return err
   436  		}
   437  
   438  		resultJSON, err := vmcompute.HcsModifyProcess(ctx, process.handle, string(modifyRequestb))
   439  		events := processHcsResult(ctx, resultJSON)
   440  		if err != nil {
   441  			return makeProcessError(process, operation, err, events)
   442  		}
   443  	}
   444  
   445  	process.stdioLock.Lock()
   446  	defer process.stdioLock.Unlock()
   447  	if process.stdin != nil {
   448  		process.stdin.Close()
   449  		process.stdin = nil
   450  	}
   451  
   452  	return nil
   453  }
   454  
   455  func (process *Process) CloseStdout(ctx context.Context) (err error) {
   456  	ctx, span := oc.StartSpan(ctx, "hcs::Process::CloseStdout") //nolint:ineffassign,staticcheck
   457  	defer span.End()
   458  	defer func() { oc.SetSpanStatus(span, err) }()
   459  	span.AddAttributes(
   460  		trace.StringAttribute("cid", process.SystemID()),
   461  		trace.Int64Attribute("pid", int64(process.processID)))
   462  
   463  	process.handleLock.Lock()
   464  	defer process.handleLock.Unlock()
   465  
   466  	if process.handle == 0 {
   467  		return nil
   468  	}
   469  
   470  	process.stdioLock.Lock()
   471  	defer process.stdioLock.Unlock()
   472  	if process.stdout != nil {
   473  		process.stdout.Close()
   474  		process.stdout = nil
   475  	}
   476  	return nil
   477  }
   478  
   479  func (process *Process) CloseStderr(ctx context.Context) (err error) {
   480  	ctx, span := oc.StartSpan(ctx, "hcs::Process::CloseStderr") //nolint:ineffassign,staticcheck
   481  	defer span.End()
   482  	defer func() { oc.SetSpanStatus(span, err) }()
   483  	span.AddAttributes(
   484  		trace.StringAttribute("cid", process.SystemID()),
   485  		trace.Int64Attribute("pid", int64(process.processID)))
   486  
   487  	process.handleLock.Lock()
   488  	defer process.handleLock.Unlock()
   489  
   490  	if process.handle == 0 {
   491  		return nil
   492  	}
   493  
   494  	process.stdioLock.Lock()
   495  	defer process.stdioLock.Unlock()
   496  	if process.stderr != nil {
   497  		process.stderr.Close()
   498  		process.stderr = nil
   499  	}
   500  	return nil
   501  }
   502  
   503  // Close cleans up any state associated with the process but does not kill
   504  // or wait on it.
   505  func (process *Process) Close() (err error) {
   506  	operation := "hcs::Process::Close"
   507  	ctx, span := oc.StartSpan(context.Background(), operation)
   508  	defer span.End()
   509  	defer func() { oc.SetSpanStatus(span, err) }()
   510  	span.AddAttributes(
   511  		trace.StringAttribute("cid", process.SystemID()),
   512  		trace.Int64Attribute("pid", int64(process.processID)))
   513  
   514  	process.handleLock.Lock()
   515  	defer process.handleLock.Unlock()
   516  
   517  	// Don't double free this
   518  	if process.handle == 0 {
   519  		return nil
   520  	}
   521  
   522  	process.stdioLock.Lock()
   523  	if process.stdin != nil {
   524  		process.stdin.Close()
   525  		process.stdin = nil
   526  	}
   527  	if process.stdout != nil {
   528  		process.stdout.Close()
   529  		process.stdout = nil
   530  	}
   531  	if process.stderr != nil {
   532  		process.stderr.Close()
   533  		process.stderr = nil
   534  	}
   535  	process.stdioLock.Unlock()
   536  
   537  	if err = process.unregisterCallback(ctx); err != nil {
   538  		return makeProcessError(process, operation, err, nil)
   539  	}
   540  
   541  	if err = vmcompute.HcsCloseProcess(ctx, process.handle); err != nil {
   542  		return makeProcessError(process, operation, err, nil)
   543  	}
   544  
   545  	process.handle = 0
   546  	process.closedWaitOnce.Do(func() {
   547  		process.exitCode = -1
   548  		process.waitError = ErrAlreadyClosed
   549  		close(process.waitBlock)
   550  	})
   551  
   552  	return nil
   553  }
   554  
   555  func (process *Process) registerCallback(ctx context.Context) error {
   556  	callbackContext := &notificationWatcherContext{
   557  		channels:  newProcessChannels(),
   558  		systemID:  process.SystemID(),
   559  		processID: process.processID,
   560  	}
   561  
   562  	callbackMapLock.Lock()
   563  	callbackNumber := nextCallback
   564  	nextCallback++
   565  	callbackMap[callbackNumber] = callbackContext
   566  	callbackMapLock.Unlock()
   567  
   568  	callbackHandle, err := vmcompute.HcsRegisterProcessCallback(ctx, process.handle, notificationWatcherCallback, callbackNumber)
   569  	if err != nil {
   570  		return err
   571  	}
   572  	callbackContext.handle = callbackHandle
   573  	process.callbackNumber = callbackNumber
   574  
   575  	return nil
   576  }
   577  
   578  func (process *Process) unregisterCallback(ctx context.Context) error {
   579  	callbackNumber := process.callbackNumber
   580  
   581  	callbackMapLock.RLock()
   582  	callbackContext := callbackMap[callbackNumber]
   583  	callbackMapLock.RUnlock()
   584  
   585  	if callbackContext == nil {
   586  		return nil
   587  	}
   588  
   589  	handle := callbackContext.handle
   590  
   591  	if handle == 0 {
   592  		return nil
   593  	}
   594  
   595  	// vmcompute.HcsUnregisterProcessCallback has its own synchronization to
   596  	// wait for all callbacks to complete. We must NOT hold the callbackMapLock.
   597  	err := vmcompute.HcsUnregisterProcessCallback(ctx, handle)
   598  	if err != nil {
   599  		return err
   600  	}
   601  
   602  	closeChannels(callbackContext.channels)
   603  
   604  	callbackMapLock.Lock()
   605  	delete(callbackMap, callbackNumber)
   606  	callbackMapLock.Unlock()
   607  
   608  	handle = 0 //nolint:ineffassign
   609  
   610  	return nil
   611  }
   612
View as plain text