container.go

Documentation: github.com/Microsoft/hcsshim/internal/guest/runtime/runc

     1  //go:build linux
     2  // +build linux
     3  
     4  package runc
     5  
     6  import (
     7  	"encoding/json"
     8  	"fmt"
     9  	"net"
    10  	"os"
    11  	"path/filepath"
    12  	"strconv"
    13  	"strings"
    14  	"syscall"
    15  
    16  	oci "github.com/opencontainers/runtime-spec/specs-go"
    17  	"github.com/pkg/errors"
    18  	"github.com/sirupsen/logrus"
    19  	"golang.org/x/sys/unix"
    20  
    21  	"github.com/Microsoft/hcsshim/internal/guest/runtime"
    22  	"github.com/Microsoft/hcsshim/internal/guest/stdio"
    23  	"github.com/Microsoft/hcsshim/internal/logfields"
    24  )
    25  
    26  type container struct {
    27  	r    *runcRuntime
    28  	id   string
    29  	init *process
    30  	// ownsPidNamespace indicates whether the container's init process is also
    31  	// the init process for its pid namespace.
    32  	ownsPidNamespace bool
    33  }
    34  
    35  var _ runtime.Container = &container{}
    36  
    37  func (c *container) ID() string {
    38  	return c.id
    39  }
    40  
    41  func (c *container) Pid() int {
    42  	return c.init.Pid()
    43  }
    44  
    45  func (c *container) Tty() *stdio.TtyRelay {
    46  	return c.init.ttyRelay
    47  }
    48  
    49  func (c *container) PipeRelay() *stdio.PipeRelay {
    50  	return c.init.pipeRelay
    51  }
    52  
    53  // Start unblocks the container's init process created by the call to
    54  // CreateContainer.
    55  func (c *container) Start() error {
    56  	logPath := c.r.getLogPath(c.id)
    57  	args := []string{"start", c.id}
    58  	cmd := runcCommandLog(logPath, args...)
    59  	out, err := cmd.CombinedOutput()
    60  	if err != nil {
    61  		runcErr := getRuncLogError(logPath)
    62  		c.r.cleanupContainer(c.id) //nolint:errcheck
    63  		return errors.Wrapf(runcErr, "runc start failed with %v: %s", err, string(out))
    64  	}
    65  	return nil
    66  }
    67  
    68  // ExecProcess executes a new process, represented as an OCI process struct,
    69  // inside an already-running container.
    70  func (c *container) ExecProcess(process *oci.Process, stdioSet *stdio.ConnectionSet) (p runtime.Process, err error) {
    71  	p, err = c.runExecCommand(process, stdioSet)
    72  	if err != nil {
    73  		return nil, err
    74  	}
    75  	return p, nil
    76  }
    77  
    78  // Kill sends the specified signal to the container's init process.
    79  func (c *container) Kill(signal syscall.Signal) error {
    80  	logrus.WithField(logfields.ContainerID, c.id).Debug("runc::container::Kill")
    81  	args := []string{"kill"}
    82  	if signal == syscall.SIGTERM || signal == syscall.SIGKILL {
    83  		args = append(args, "--all")
    84  	}
    85  	args = append(args, c.id, strconv.Itoa(int(signal)))
    86  	cmd := runcCommand(args...)
    87  	out, err := cmd.CombinedOutput()
    88  	if err != nil {
    89  		runcErr := parseRuncError(string(out))
    90  		return errors.Wrapf(runcErr, "unknown runc error after kill %v: %s", err, string(out))
    91  	}
    92  	return nil
    93  }
    94  
    95  // Delete deletes any state created for the container by either this wrapper or
    96  // runC itself.
    97  func (c *container) Delete() error {
    98  	logrus.WithField(logfields.ContainerID, c.id).Debug("runc::container::Delete")
    99  	cmd := runcCommand("delete", c.id)
   100  	out, err := cmd.CombinedOutput()
   101  	if err != nil {
   102  		runcErr := parseRuncError(string(out))
   103  		return errors.Wrapf(runcErr, "runc delete failed with %v: %s", err, string(out))
   104  	}
   105  	return c.r.cleanupContainer(c.id)
   106  }
   107  
   108  // Pause suspends all processes running in the container.
   109  func (c *container) Pause() error {
   110  	cmd := runcCommand("pause", c.id)
   111  	out, err := cmd.CombinedOutput()
   112  	if err != nil {
   113  		runcErr := parseRuncError(string(out))
   114  		return errors.Wrapf(runcErr, "runc pause failed with %v: %s", err, string(out))
   115  	}
   116  	return nil
   117  }
   118  
   119  // Resume unsuspends processes running in the container.
   120  func (c *container) Resume() error {
   121  	logPath := c.r.getLogPath(c.id)
   122  	args := []string{"resume", c.id}
   123  	cmd := runcCommandLog(logPath, args...)
   124  	out, err := cmd.CombinedOutput()
   125  	if err != nil {
   126  		runcErr := getRuncLogError(logPath)
   127  		return errors.Wrapf(runcErr, "runc resume failed with %v: %s", err, string(out))
   128  	}
   129  	return nil
   130  }
   131  
   132  // GetState returns information about the given container.
   133  func (c *container) GetState() (*runtime.ContainerState, error) {
   134  	cmd := runcCommand("state", c.id)
   135  	out, err := cmd.CombinedOutput()
   136  	if err != nil {
   137  		runcErr := parseRuncError(string(out))
   138  		return nil, errors.Wrapf(runcErr, "runc state failed with %v: %s", err, string(out))
   139  	}
   140  	var state runtime.ContainerState
   141  	if err := json.Unmarshal(out, &state); err != nil {
   142  		return nil, errors.Wrapf(err, "failed to unmarshal the state for container %s", c.id)
   143  	}
   144  	return &state, nil
   145  }
   146  
   147  // Exists returns true if the container exists, false if it doesn't
   148  // exist.
   149  // It should be noted that containers that have stopped but have not been
   150  // deleted are still considered to exist.
   151  func (c *container) Exists() (bool, error) {
   152  	// use global path because container may not exist
   153  	cmd := runcCommand("state", c.id)
   154  	out, err := cmd.CombinedOutput()
   155  	if err != nil {
   156  		runcErr := parseRuncError(string(out))
   157  		if errors.Is(runcErr, runtime.ErrContainerDoesNotExist) {
   158  			return false, nil
   159  		}
   160  		return false, errors.Wrapf(runcErr, "runc state failed with %v: %s", err, string(out))
   161  	}
   162  	return true, nil
   163  }
   164  
   165  // GetRunningProcesses gets only the running processes associated with the given
   166  // container. This excludes zombie processes.
   167  func (c *container) GetRunningProcesses() ([]runtime.ContainerProcessState, error) {
   168  	pids, err := c.r.getRunningPids(c.id)
   169  	if err != nil {
   170  		return nil, err
   171  	}
   172  
   173  	pidMap := map[int]*runtime.ContainerProcessState{}
   174  	// Initialize all processes with a pid and command, and mark correctly that
   175  	// none of them are zombies. Default CreatedByRuntime to false.
   176  	for _, pid := range pids {
   177  		command, err := c.r.getProcessCommand(pid)
   178  		if err != nil {
   179  			if errors.Is(err, unix.ENOENT) {
   180  				// process has exited between getting the running pids above
   181  				// and now, ignore error
   182  				continue
   183  			}
   184  			return nil, err
   185  		}
   186  		pidMap[pid] = &runtime.ContainerProcessState{Pid: pid, Command: command, CreatedByRuntime: false, IsZombie: false}
   187  	}
   188  
   189  	// For each process state directory which corresponds to a running pid, set
   190  	// that the process was created by the Runtime.
   191  	processDirs, err := os.ReadDir(filepath.Join(containerFilesDir, c.id))
   192  	if err != nil {
   193  		return nil, errors.Wrapf(err, "failed to read the contents of container directory %s", filepath.Join(containerFilesDir, c.id))
   194  	}
   195  	for _, processDir := range processDirs {
   196  		if processDir.Name() != initPidFilename {
   197  			pid, err := strconv.Atoi(processDir.Name())
   198  			if err != nil {
   199  				return nil, errors.Wrapf(err, "failed to parse string \"%s\" as pid", processDir.Name())
   200  			}
   201  			if _, ok := pidMap[pid]; ok {
   202  				pidMap[pid].CreatedByRuntime = true
   203  			}
   204  		}
   205  	}
   206  	return c.r.pidMapToProcessStates(pidMap), nil
   207  }
   208  
   209  // GetAllProcesses gets all processes associated with the given container,
   210  // including both running and zombie processes.
   211  func (c *container) GetAllProcesses() ([]runtime.ContainerProcessState, error) {
   212  	runningPids, err := c.r.getRunningPids(c.id)
   213  	if err != nil {
   214  		return nil, err
   215  	}
   216  
   217  	logrus.WithFields(logrus.Fields{
   218  		"cid":  c.id,
   219  		"pids": runningPids,
   220  	}).Debug("running container pids")
   221  
   222  	pidMap := map[int]*runtime.ContainerProcessState{}
   223  	// Initialize all processes with a pid and command, leaving CreatedByRuntime
   224  	// and IsZombie at the default value of false.
   225  	for _, pid := range runningPids {
   226  		command, err := c.r.getProcessCommand(pid)
   227  		if err != nil {
   228  			if errors.Is(err, unix.ENOENT) {
   229  				// process has exited between getting the running pids above
   230  				// and now, ignore error
   231  				continue
   232  			}
   233  			return nil, err
   234  		}
   235  		pidMap[pid] = &runtime.ContainerProcessState{Pid: pid, Command: command, CreatedByRuntime: false, IsZombie: false}
   236  	}
   237  
   238  	processDirs, err := os.ReadDir(filepath.Join(containerFilesDir, c.id))
   239  	if err != nil {
   240  		return nil, errors.Wrapf(err, "failed to read the contents of container directory %s", filepath.Join(containerFilesDir, c.id))
   241  	}
   242  	// Loop over every process state directory. Since these processes have
   243  	// process state directories, CreatedByRuntime will be true for all of them.
   244  	for _, processDir := range processDirs {
   245  		if processDir.Name() != initPidFilename {
   246  			pid, err := strconv.Atoi(processDir.Name())
   247  			if err != nil {
   248  				return nil, errors.Wrapf(err, "failed to parse string \"%s\" into pid", processDir.Name())
   249  			}
   250  			if c.r.processExists(pid) {
   251  				// If the process exists in /proc and is in the pidMap, it must
   252  				// be a running non-zombie.
   253  				if _, ok := pidMap[pid]; ok {
   254  					pidMap[pid].CreatedByRuntime = true
   255  				} else {
   256  					// Otherwise, since it's in /proc but not running, it must
   257  					// be a zombie.
   258  					command, err := c.r.getProcessCommand(pid)
   259  					if err != nil {
   260  						if errors.Is(err, unix.ENOENT) {
   261  							// process has exited between checking that it exists and now, ignore error
   262  							continue
   263  						}
   264  						return nil, err
   265  					}
   266  					pidMap[pid] = &runtime.ContainerProcessState{Pid: pid, Command: command, CreatedByRuntime: true, IsZombie: true}
   267  				}
   268  			}
   269  		}
   270  	}
   271  	return c.r.pidMapToProcessStates(pidMap), nil
   272  }
   273  
   274  // GetInitProcess gets the init processes associated with the given container,
   275  // including both running and zombie processes.
   276  func (c *container) GetInitProcess() (runtime.Process, error) {
   277  	if c.init == nil {
   278  		return nil, errors.New("container has no init process")
   279  	}
   280  	return c.init, nil
   281  }
   282  
   283  // Wait waits on every non-init process in the container, and then performs a
   284  // final wait on the init process. The exit code returned is the exit code
   285  // acquired from waiting on the init process.
   286  func (c *container) Wait() (int, error) {
   287  	entity := logrus.WithField(logfields.ContainerID, c.id)
   288  	processes, err := c.GetAllProcesses()
   289  	if err != nil {
   290  		return -1, err
   291  	}
   292  	for _, process := range processes {
   293  		// Only wait on non-init processes that were created with exec.
   294  		if process.Pid != c.init.pid && process.CreatedByRuntime {
   295  			// FUTURE-jstarks: Consider waiting on the child process's relays as
   296  			// well (as in p.Wait()). This may not matter as long as the relays
   297  			// finish "soon" after Wait() returns since HCS expects the stdio
   298  			// connections to close before container shutdown can complete.
   299  			entity.WithField(logfields.ProcessID, process.Pid).Debug("waiting on container exec process")
   300  			_, _ = c.r.waitOnProcess(process.Pid)
   301  		}
   302  	}
   303  	exitCode, err := c.init.Wait()
   304  	entity.Debug("runc::container::init process wait completed")
   305  	if err != nil {
   306  		return -1, err
   307  	}
   308  	return exitCode, nil
   309  }
   310  
   311  // runExecCommand sets up the arguments for calling runc exec.
   312  func (c *container) runExecCommand(processDef *oci.Process, stdioSet *stdio.ConnectionSet) (p runtime.Process, err error) {
   313  	// Create a temporary random directory to store the process's files.
   314  	tempProcessDir, err := os.MkdirTemp(containerFilesDir, c.id)
   315  	if err != nil {
   316  		return nil, err
   317  	}
   318  
   319  	f, err := os.Create(filepath.Join(tempProcessDir, "process.json"))
   320  	if err != nil {
   321  		return nil, errors.Wrapf(err, "failed to create process.json file at %s", filepath.Join(tempProcessDir, "process.json"))
   322  	}
   323  	defer f.Close()
   324  	if err := json.NewEncoder(f).Encode(processDef); err != nil {
   325  		return nil, errors.Wrap(err, "failed to encode JSON into process.json file")
   326  	}
   327  
   328  	args := []string{"exec"}
   329  	args = append(args, "-d", "--process", filepath.Join(tempProcessDir, "process.json"))
   330  	return c.startProcess(tempProcessDir, processDef.Terminal, stdioSet, nil, args...)
   331  }
   332  
   333  // startProcess performs the operations necessary to start a container process
   334  // and properly handle its stdio. This function is used by both CreateContainer
   335  // and ExecProcess. For V2 container creation stdioSet will be nil, in this case
   336  // it is expected that the caller starts the relay previous to calling Start on
   337  // the container.
   338  func (c *container) startProcess(
   339  	tempProcessDir string,
   340  	hasTerminal bool,
   341  	stdioSet *stdio.ConnectionSet,
   342  	annotations map[string]string,
   343  	initialArgs ...string,
   344  ) (p *process, err error) {
   345  	args := initialArgs
   346  
   347  	if err := setSubreaper(1); err != nil {
   348  		return nil, errors.Wrapf(err, "failed to set process as subreaper for process in container %s", c.id)
   349  	}
   350  	if err := c.r.makeLogDir(c.id); err != nil {
   351  		return nil, err
   352  	}
   353  
   354  	logPath := c.r.getLogPath(c.id)
   355  	args = append(args, "--pid-file", filepath.Join(tempProcessDir, "pid"))
   356  
   357  	var sockListener *net.UnixListener
   358  	if hasTerminal {
   359  		var consoleSockPath string
   360  		sockListener, consoleSockPath, err = c.r.createConsoleSocket(tempProcessDir)
   361  		if err != nil {
   362  			return nil, errors.Wrapf(err, "failed to create console socket for container %s", c.id)
   363  		}
   364  		defer sockListener.Close()
   365  		args = append(args, "--console-socket", consoleSockPath)
   366  	}
   367  	args = append(args, c.id)
   368  
   369  	cmd := runcCommandLog(logPath, args...)
   370  
   371  	var pipeRelay *stdio.PipeRelay
   372  	if !hasTerminal {
   373  		pipeRelay, err = stdio.NewPipeRelay(stdioSet)
   374  		if err != nil {
   375  			return nil, errors.Wrapf(err, "failed to create a pipe relay connection set for container %s", c.id)
   376  		}
   377  		fileSet, err := pipeRelay.Files()
   378  		if err != nil {
   379  			return nil, errors.Wrapf(err, "failed to get files for connection set for container %s", c.id)
   380  		}
   381  		// Closing the FileSet here is fine as that end of the pipes will have
   382  		// already been copied into the child process.
   383  		defer fileSet.Close()
   384  		if fileSet.In != nil {
   385  			cmd.Stdin = fileSet.In
   386  		}
   387  		if fileSet.Out != nil {
   388  			cmd.Stdout = fileSet.Out
   389  		}
   390  		if fileSet.Err != nil {
   391  			cmd.Stderr = fileSet.Err
   392  		}
   393  	}
   394  
   395  	// This is for enabling container logging via side car feature. This is in experimental stage now and need to be revisited.
   396  	var stdoutFifoPipe, stderrFifoPipe *os.File
   397  	if annotations != nil {
   398  		pipeNameSuffix, exists := annotations["io.microsoft.bmc.logging.pipelocation"]
   399  		if exists {
   400  			if hasTerminal {
   401  				return nil, fmt.Errorf("logging via side car and TTY are not supported together")
   402  			}
   403  			pipeDirectory := "/run/gcs/containerlogs/"
   404  			stdoutPipeName := pipeDirectory + pipeNameSuffix + "-stdout"
   405  			stderrPipeName := pipeDirectory + pipeNameSuffix + "-stderr"
   406  			err = os.MkdirAll(pipeDirectory, 0755)
   407  			if err != nil {
   408  				return nil, fmt.Errorf("error creating log directory %s for logging pipe fifo: %w", pipeDirectory, err)
   409  			}
   410  
   411  			_, err = os.Stat(stdoutPipeName)
   412  			if err != nil {
   413  				// fifo pipe does not exist, create one
   414  				err = syscall.Mkfifo(stdoutPipeName, 0666)
   415  				if err != nil {
   416  					return nil, fmt.Errorf("error creating fifo %s: %w", stdoutPipeName, err)
   417  				}
   418  			}
   419  
   420  			_, err = os.Stat(stderrPipeName)
   421  			if err != nil {
   422  				// fifo pipe does not exist, create one
   423  				err = syscall.Mkfifo(stderrPipeName, 0666)
   424  				if err != nil {
   425  					return nil, fmt.Errorf("error creating fifo %s: %w", stderrPipeName, err)
   426  				}
   427  			}
   428  
   429  			// pipe either exist before hand or we have created one above
   430  			stdoutFifoPipe, err = os.OpenFile(stdoutPipeName, os.O_RDWR|os.O_APPEND, os.ModeNamedPipe)
   431  			if err != nil {
   432  				return nil, fmt.Errorf("error opening fifo %s: %w", stdoutPipeName, err)
   433  			}
   434  
   435  			stderrFifoPipe, err = os.OpenFile(stderrPipeName, os.O_RDWR|os.O_APPEND, os.ModeNamedPipe)
   436  			if err != nil {
   437  				return nil, fmt.Errorf("error opening fifo %s: %w", stderrPipeName, err)
   438  			}
   439  		}
   440  
   441  		isLoggingSideCarContainerStr, exists := annotations["io.microsoft.bmc.logging.isLoggingSideCarContainer"]
   442  		if exists {
   443  			isLoggingSideCarContainer, err := strconv.ParseBool(isLoggingSideCarContainerStr)
   444  			if err != nil {
   445  				return nil, fmt.Errorf("error parsing flag isLoggingSideCarContainer: %w", err)
   446  			}
   447  			if !isLoggingSideCarContainer {
   448  				// workload container needs to redirect stdout and stderr to fifo pipe.
   449  				cmd.Stdout = stdoutFifoPipe
   450  				cmd.Stderr = stderrFifoPipe
   451  			} else {
   452  				// logging side car container needs to know the pipe fd.
   453  				cmd.Args = append(cmd.Args, "--preserve-fds", "2")
   454  				cmd.ExtraFiles = []*os.File{stdoutFifoPipe, stderrFifoPipe}
   455  			}
   456  		}
   457  	}
   458  
   459  	if err := cmd.Run(); err != nil {
   460  		runcErr := getRuncLogError(logPath)
   461  		return nil, errors.Wrapf(runcErr, "failed to run runc create/exec call for container %s with %v", c.id, err)
   462  	}
   463  
   464  	var ttyRelay *stdio.TtyRelay
   465  	if hasTerminal {
   466  		var master *os.File
   467  		master, err = c.r.getMasterFromSocket(sockListener)
   468  		if err != nil {
   469  			_ = cmd.Process.Kill()
   470  			return nil, errors.Wrapf(err, "failed to get pty master for process in container %s", c.id)
   471  		}
   472  		// Keep master open for the relay unless there is an error.
   473  		defer func() {
   474  			if err != nil {
   475  				master.Close()
   476  			}
   477  		}()
   478  		ttyRelay = stdio.NewTtyRelay(stdioSet, master)
   479  	}
   480  
   481  	// Rename the process's directory to its pid.
   482  	pid, err := c.r.readPidFile(filepath.Join(tempProcessDir, "pid"))
   483  	if err != nil {
   484  		return nil, err
   485  	}
   486  	if err := os.Rename(tempProcessDir, c.r.getProcessDir(c.id, pid)); err != nil {
   487  		return nil, err
   488  	}
   489  
   490  	if ttyRelay != nil && stdioSet != nil {
   491  		ttyRelay.Start()
   492  	}
   493  	if pipeRelay != nil && stdioSet != nil {
   494  		pipeRelay.Start()
   495  	}
   496  	return &process{c: c, pid: pid, ttyRelay: ttyRelay, pipeRelay: pipeRelay}, nil
   497  }
   498  
   499  func (c *container) Update(resources interface{}) error {
   500  	jsonResources, err := json.Marshal(resources)
   501  	if err != nil {
   502  		return err
   503  	}
   504  	cmd := runcCommand("update", "--resources", "-", c.id)
   505  	cmd.Stdin = strings.NewReader(string(jsonResources))
   506  	out, err := cmd.CombinedOutput()
   507  	if err != nil {
   508  		runcErr := parseRuncError(string(out))
   509  		return errors.Wrapf(runcErr, "runc update request %s failed with %v: %s", string(jsonResources), err, string(out))
   510  	}
   511  	return nil
   512  }
   513
View as plain text