Source file src/github.com/opencontainers/runc/libcontainer/integration/exec_test.go

Documentation: github.com/opencontainers/runc/libcontainer/integration

     1  package integration
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/json"
     6  	"errors"
     7  	"fmt"
     8  	"os"
     9  	"os/exec"
    10  	"path/filepath"
    11  	"reflect"
    12  	"strconv"
    13  	"strings"
    14  	"syscall"
    15  	"testing"
    16  
    17  	"github.com/opencontainers/runc/libcontainer"
    18  	"github.com/opencontainers/runc/libcontainer/cgroups"
    19  	"github.com/opencontainers/runc/libcontainer/cgroups/systemd"
    20  	"github.com/opencontainers/runc/libcontainer/configs"
    21  	"github.com/opencontainers/runc/libcontainer/userns"
    22  	"github.com/opencontainers/runtime-spec/specs-go"
    23  
    24  	"golang.org/x/sys/unix"
    25  )
    26  
    27  func TestExecPS(t *testing.T) {
    28  	testExecPS(t, false)
    29  }
    30  
    31  func TestUsernsExecPS(t *testing.T) {
    32  	if _, err := os.Stat("/proc/self/ns/user"); os.IsNotExist(err) {
    33  		t.Skip("Test requires userns.")
    34  	}
    35  	testExecPS(t, true)
    36  }
    37  
    38  func testExecPS(t *testing.T, userns bool) {
    39  	if testing.Short() {
    40  		return
    41  	}
    42  	config := newTemplateConfig(t, &tParam{userns: userns})
    43  
    44  	buffers := runContainerOk(t, config, "ps", "-o", "pid,user,comm")
    45  	lines := strings.Split(buffers.Stdout.String(), "\n")
    46  	if len(lines) < 2 {
    47  		t.Fatalf("more than one process running for output %q", buffers.Stdout.String())
    48  	}
    49  	expected := `1 root     ps`
    50  	actual := strings.Trim(lines[1], "\n ")
    51  	if actual != expected {
    52  		t.Fatalf("expected output %q but received %q", expected, actual)
    53  	}
    54  }
    55  
    56  func TestIPCPrivate(t *testing.T) {
    57  	if testing.Short() {
    58  		return
    59  	}
    60  
    61  	l, err := os.Readlink("/proc/1/ns/ipc")
    62  	ok(t, err)
    63  
    64  	config := newTemplateConfig(t, nil)
    65  	buffers := runContainerOk(t, config, "readlink", "/proc/self/ns/ipc")
    66  
    67  	if actual := strings.Trim(buffers.Stdout.String(), "\n"); actual == l {
    68  		t.Fatalf("ipc link should be private to the container but equals host %q %q", actual, l)
    69  	}
    70  }
    71  
    72  func TestIPCHost(t *testing.T) {
    73  	if testing.Short() {
    74  		return
    75  	}
    76  
    77  	l, err := os.Readlink("/proc/1/ns/ipc")
    78  	ok(t, err)
    79  
    80  	config := newTemplateConfig(t, nil)
    81  	config.Namespaces.Remove(configs.NEWIPC)
    82  	buffers := runContainerOk(t, config, "readlink", "/proc/self/ns/ipc")
    83  
    84  	if actual := strings.Trim(buffers.Stdout.String(), "\n"); actual != l {
    85  		t.Fatalf("ipc link not equal to host link %q %q", actual, l)
    86  	}
    87  }
    88  
    89  func TestIPCJoinPath(t *testing.T) {
    90  	if testing.Short() {
    91  		return
    92  	}
    93  
    94  	l, err := os.Readlink("/proc/1/ns/ipc")
    95  	ok(t, err)
    96  
    97  	config := newTemplateConfig(t, nil)
    98  	config.Namespaces.Add(configs.NEWIPC, "/proc/1/ns/ipc")
    99  	buffers := runContainerOk(t, config, "readlink", "/proc/self/ns/ipc")
   100  
   101  	if actual := strings.Trim(buffers.Stdout.String(), "\n"); actual != l {
   102  		t.Fatalf("ipc link not equal to host link %q %q", actual, l)
   103  	}
   104  }
   105  
   106  func TestIPCBadPath(t *testing.T) {
   107  	if testing.Short() {
   108  		return
   109  	}
   110  
   111  	config := newTemplateConfig(t, nil)
   112  	config.Namespaces.Add(configs.NEWIPC, "/proc/1/ns/ipcc")
   113  
   114  	if _, _, err := runContainer(t, config, "true"); err == nil {
   115  		t.Fatal("container succeeded with bad ipc path")
   116  	}
   117  }
   118  
   119  func TestRlimit(t *testing.T) {
   120  	testRlimit(t, false)
   121  }
   122  
   123  func TestUsernsRlimit(t *testing.T) {
   124  	if _, err := os.Stat("/proc/self/ns/user"); os.IsNotExist(err) {
   125  		t.Skip("Test requires userns.")
   126  	}
   127  
   128  	testRlimit(t, true)
   129  }
   130  
   131  func testRlimit(t *testing.T, userns bool) {
   132  	if testing.Short() {
   133  		return
   134  	}
   135  
   136  	config := newTemplateConfig(t, &tParam{userns: userns})
   137  
   138  	// ensure limit is lower than what the config requests to test that in a user namespace
   139  	// the Setrlimit call happens early enough that we still have permissions to raise the limit.
   140  	ok(t, unix.Setrlimit(unix.RLIMIT_NOFILE, &unix.Rlimit{
   141  		Max: 1024,
   142  		Cur: 1024,
   143  	}))
   144  
   145  	out := runContainerOk(t, config, "/bin/sh", "-c", "ulimit -n")
   146  	if limit := strings.TrimSpace(out.Stdout.String()); limit != "1025" {
   147  		t.Fatalf("expected rlimit to be 1025, got %s", limit)
   148  	}
   149  }
   150  
   151  func TestEnter(t *testing.T) {
   152  	if testing.Short() {
   153  		return
   154  	}
   155  
   156  	config := newTemplateConfig(t, nil)
   157  
   158  	container, err := newContainer(t, config)
   159  	ok(t, err)
   160  	defer destroyContainer(container)
   161  
   162  	// Execute a first process in the container
   163  	stdinR, stdinW, err := os.Pipe()
   164  	ok(t, err)
   165  
   166  	var stdout, stdout2 bytes.Buffer
   167  
   168  	pconfig := libcontainer.Process{
   169  		Cwd:    "/",
   170  		Args:   []string{"sh", "-c", "cat && readlink /proc/self/ns/pid"},
   171  		Env:    standardEnvironment,
   172  		Stdin:  stdinR,
   173  		Stdout: &stdout,
   174  		Init:   true,
   175  	}
   176  	err = container.Run(&pconfig)
   177  	_ = stdinR.Close()
   178  	defer stdinW.Close() //nolint: errcheck
   179  	ok(t, err)
   180  	pid, err := pconfig.Pid()
   181  	ok(t, err)
   182  
   183  	// Execute another process in the container
   184  	stdinR2, stdinW2, err := os.Pipe()
   185  	ok(t, err)
   186  	pconfig2 := libcontainer.Process{
   187  		Cwd: "/",
   188  		Env: standardEnvironment,
   189  	}
   190  	pconfig2.Args = []string{"sh", "-c", "cat && readlink /proc/self/ns/pid"}
   191  	pconfig2.Stdin = stdinR2
   192  	pconfig2.Stdout = &stdout2
   193  
   194  	err = container.Run(&pconfig2)
   195  	_ = stdinR2.Close()
   196  	defer stdinW2.Close() //nolint: errcheck
   197  	ok(t, err)
   198  
   199  	pid2, err := pconfig2.Pid()
   200  	ok(t, err)
   201  
   202  	processes, err := container.Processes()
   203  	ok(t, err)
   204  
   205  	n := 0
   206  	for i := range processes {
   207  		if processes[i] == pid || processes[i] == pid2 {
   208  			n++
   209  		}
   210  	}
   211  	if n != 2 {
   212  		t.Fatal("unexpected number of processes", processes, pid, pid2)
   213  	}
   214  
   215  	// Wait processes
   216  	_ = stdinW2.Close()
   217  	waitProcess(&pconfig2, t)
   218  
   219  	_ = stdinW.Close()
   220  	waitProcess(&pconfig, t)
   221  
   222  	// Check that both processes live in the same pidns
   223  	pidns := stdout.String()
   224  	ok(t, err)
   225  
   226  	pidns2 := stdout2.String()
   227  	ok(t, err)
   228  
   229  	if pidns != pidns2 {
   230  		t.Fatal("The second process isn't in the required pid namespace", pidns, pidns2)
   231  	}
   232  }
   233  
   234  func TestProcessEnv(t *testing.T) {
   235  	if testing.Short() {
   236  		return
   237  	}
   238  
   239  	config := newTemplateConfig(t, nil)
   240  	container, err := newContainer(t, config)
   241  	ok(t, err)
   242  	defer destroyContainer(container)
   243  
   244  	var stdout bytes.Buffer
   245  	pconfig := libcontainer.Process{
   246  		Cwd:  "/",
   247  		Args: []string{"sh", "-c", "env"},
   248  		Env: []string{
   249  			"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
   250  			"HOSTNAME=integration",
   251  			"TERM=xterm",
   252  			"FOO=BAR",
   253  		},
   254  		Stdin:  nil,
   255  		Stdout: &stdout,
   256  		Init:   true,
   257  	}
   258  	err = container.Run(&pconfig)
   259  	ok(t, err)
   260  
   261  	// Wait for process
   262  	waitProcess(&pconfig, t)
   263  
   264  	outputEnv := stdout.String()
   265  
   266  	// Check that the environment has the key/value pair we added
   267  	if !strings.Contains(outputEnv, "FOO=BAR") {
   268  		t.Fatal("Environment doesn't have the expected FOO=BAR key/value pair: ", outputEnv)
   269  	}
   270  
   271  	// Make sure that HOME is set
   272  	if !strings.Contains(outputEnv, "HOME=/root") {
   273  		t.Fatal("Environment doesn't have HOME set: ", outputEnv)
   274  	}
   275  }
   276  
   277  func TestProcessEmptyCaps(t *testing.T) {
   278  	if testing.Short() {
   279  		return
   280  	}
   281  
   282  	config := newTemplateConfig(t, nil)
   283  	config.Capabilities = nil
   284  
   285  	container, err := newContainer(t, config)
   286  	ok(t, err)
   287  	defer destroyContainer(container)
   288  
   289  	var stdout bytes.Buffer
   290  	pconfig := libcontainer.Process{
   291  		Cwd:    "/",
   292  		Args:   []string{"sh", "-c", "cat /proc/self/status"},
   293  		Env:    standardEnvironment,
   294  		Stdin:  nil,
   295  		Stdout: &stdout,
   296  		Init:   true,
   297  	}
   298  	err = container.Run(&pconfig)
   299  	ok(t, err)
   300  
   301  	// Wait for process
   302  	waitProcess(&pconfig, t)
   303  
   304  	outputStatus := stdout.String()
   305  
   306  	lines := strings.Split(outputStatus, "\n")
   307  
   308  	effectiveCapsLine := ""
   309  	for _, l := range lines {
   310  		line := strings.TrimSpace(l)
   311  		if strings.Contains(line, "CapEff:") {
   312  			effectiveCapsLine = line
   313  			break
   314  		}
   315  	}
   316  
   317  	if effectiveCapsLine == "" {
   318  		t.Fatal("Couldn't find effective caps: ", outputStatus)
   319  	}
   320  }
   321  
   322  func TestProcessCaps(t *testing.T) {
   323  	if testing.Short() {
   324  		return
   325  	}
   326  
   327  	config := newTemplateConfig(t, nil)
   328  	container, err := newContainer(t, config)
   329  	ok(t, err)
   330  	defer destroyContainer(container)
   331  
   332  	var stdout bytes.Buffer
   333  	pconfig := libcontainer.Process{
   334  		Cwd:          "/",
   335  		Args:         []string{"sh", "-c", "cat /proc/self/status"},
   336  		Env:          standardEnvironment,
   337  		Stdin:        nil,
   338  		Stdout:       &stdout,
   339  		Capabilities: &configs.Capabilities{},
   340  		Init:         true,
   341  	}
   342  	pconfig.Capabilities.Bounding = append(config.Capabilities.Bounding, "CAP_NET_ADMIN")
   343  	pconfig.Capabilities.Permitted = append(config.Capabilities.Permitted, "CAP_NET_ADMIN")
   344  	pconfig.Capabilities.Effective = append(config.Capabilities.Effective, "CAP_NET_ADMIN")
   345  	err = container.Run(&pconfig)
   346  	ok(t, err)
   347  
   348  	// Wait for process
   349  	waitProcess(&pconfig, t)
   350  
   351  	outputStatus := stdout.String()
   352  
   353  	lines := strings.Split(outputStatus, "\n")
   354  
   355  	effectiveCapsLine := ""
   356  	for _, l := range lines {
   357  		line := strings.TrimSpace(l)
   358  		if strings.Contains(line, "CapEff:") {
   359  			effectiveCapsLine = line
   360  			break
   361  		}
   362  	}
   363  
   364  	if effectiveCapsLine == "" {
   365  		t.Fatal("Couldn't find effective caps: ", outputStatus)
   366  	}
   367  
   368  	parts := strings.Split(effectiveCapsLine, ":")
   369  	effectiveCapsStr := strings.TrimSpace(parts[1])
   370  
   371  	effectiveCaps, err := strconv.ParseUint(effectiveCapsStr, 16, 64)
   372  	if err != nil {
   373  		t.Fatal("Could not parse effective caps", err)
   374  	}
   375  
   376  	const netAdminMask = 1 << unix.CAP_NET_ADMIN
   377  	if effectiveCaps&netAdminMask != netAdminMask {
   378  		t.Fatal("CAP_NET_ADMIN is not set as expected")
   379  	}
   380  }
   381  
   382  func TestAdditionalGroups(t *testing.T) {
   383  	if testing.Short() {
   384  		return
   385  	}
   386  
   387  	config := newTemplateConfig(t, nil)
   388  	container, err := newContainer(t, config)
   389  	ok(t, err)
   390  	defer destroyContainer(container)
   391  
   392  	var stdout bytes.Buffer
   393  	pconfig := libcontainer.Process{
   394  		Cwd:              "/",
   395  		Args:             []string{"sh", "-c", "id", "-Gn"},
   396  		Env:              standardEnvironment,
   397  		Stdin:            nil,
   398  		Stdout:           &stdout,
   399  		AdditionalGroups: []string{"plugdev", "audio"},
   400  		Init:             true,
   401  	}
   402  	err = container.Run(&pconfig)
   403  	ok(t, err)
   404  
   405  	// Wait for process
   406  	waitProcess(&pconfig, t)
   407  
   408  	outputGroups := stdout.String()
   409  
   410  	// Check that the groups output has the groups that we specified
   411  	if !strings.Contains(outputGroups, "audio") {
   412  		t.Fatalf("Listed groups do not contain the audio group as expected: %v", outputGroups)
   413  	}
   414  
   415  	if !strings.Contains(outputGroups, "plugdev") {
   416  		t.Fatalf("Listed groups do not contain the plugdev group as expected: %v", outputGroups)
   417  	}
   418  }
   419  
   420  func TestFreeze(t *testing.T) {
   421  	for _, systemd := range []bool{true, false} {
   422  		for _, set := range []bool{true, false} {
   423  			name := ""
   424  			if systemd {
   425  				name += "Systemd"
   426  			} else {
   427  				name += "FS"
   428  			}
   429  			if set {
   430  				name += "ViaSet"
   431  			} else {
   432  				name += "ViaPauseResume"
   433  			}
   434  			t.Run(name, func(t *testing.T) {
   435  				testFreeze(t, systemd, set)
   436  			})
   437  		}
   438  	}
   439  }
   440  
   441  func testFreeze(t *testing.T, withSystemd bool, useSet bool) {
   442  	if testing.Short() {
   443  		return
   444  	}
   445  	if withSystemd && !systemd.IsRunningSystemd() {
   446  		t.Skip("Test requires systemd.")
   447  	}
   448  
   449  	config := newTemplateConfig(t, &tParam{systemd: withSystemd})
   450  	container, err := newContainer(t, config)
   451  	ok(t, err)
   452  	defer destroyContainer(container)
   453  
   454  	stdinR, stdinW, err := os.Pipe()
   455  	ok(t, err)
   456  
   457  	pconfig := &libcontainer.Process{
   458  		Cwd:   "/",
   459  		Args:  []string{"cat"},
   460  		Env:   standardEnvironment,
   461  		Stdin: stdinR,
   462  		Init:  true,
   463  	}
   464  	err = container.Run(pconfig)
   465  	_ = stdinR.Close()
   466  	defer stdinW.Close() //nolint: errcheck
   467  	ok(t, err)
   468  
   469  	if !useSet {
   470  		err = container.Pause()
   471  	} else {
   472  		config.Cgroups.Resources.Freezer = configs.Frozen
   473  		err = container.Set(*config)
   474  	}
   475  	ok(t, err)
   476  
   477  	state, err := container.Status()
   478  	ok(t, err)
   479  	if state != libcontainer.Paused {
   480  		t.Fatal("Unexpected state: ", state)
   481  	}
   482  
   483  	if !useSet {
   484  		err = container.Resume()
   485  	} else {
   486  		config.Cgroups.Resources.Freezer = configs.Thawed
   487  		err = container.Set(*config)
   488  	}
   489  	ok(t, err)
   490  
   491  	_ = stdinW.Close()
   492  	waitProcess(pconfig, t)
   493  }
   494  
   495  func TestCpuShares(t *testing.T) {
   496  	testCpuShares(t, false)
   497  }
   498  
   499  func TestCpuSharesSystemd(t *testing.T) {
   500  	if !systemd.IsRunningSystemd() {
   501  		t.Skip("Test requires systemd.")
   502  	}
   503  	testCpuShares(t, true)
   504  }
   505  
   506  func testCpuShares(t *testing.T, systemd bool) {
   507  	if testing.Short() {
   508  		return
   509  	}
   510  	if cgroups.IsCgroup2UnifiedMode() {
   511  		t.Skip("cgroup v2 does not support CpuShares")
   512  	}
   513  
   514  	config := newTemplateConfig(t, &tParam{systemd: systemd})
   515  	config.Cgroups.Resources.CpuShares = 1
   516  
   517  	if _, _, err := runContainer(t, config, "ps"); err == nil {
   518  		t.Fatal("runContainer should fail with invalid CpuShares")
   519  	}
   520  }
   521  
   522  func TestPids(t *testing.T) {
   523  	testPids(t, false)
   524  }
   525  
   526  func TestPidsSystemd(t *testing.T) {
   527  	if !systemd.IsRunningSystemd() {
   528  		t.Skip("Test requires systemd.")
   529  	}
   530  	testPids(t, true)
   531  }
   532  
   533  func testPids(t *testing.T, systemd bool) {
   534  	if testing.Short() {
   535  		return
   536  	}
   537  
   538  	config := newTemplateConfig(t, &tParam{systemd: systemd})
   539  	config.Cgroups.Resources.PidsLimit = -1
   540  
   541  	// Running multiple processes, expecting it to succeed with no pids limit.
   542  	_ = runContainerOk(t, config, "/bin/sh", "-c", "/bin/true | /bin/true | /bin/true | /bin/true")
   543  
   544  	// Enforce a permissive limit. This needs to be fairly hand-wavey due to the
   545  	// issues with running Go binaries with pids restrictions (see below).
   546  	config.Cgroups.Resources.PidsLimit = 64
   547  	_ = runContainerOk(t, config, "/bin/sh", "-c", `
   548  	/bin/true | /bin/true | /bin/true | /bin/true | /bin/true | /bin/true | bin/true | /bin/true |
   549  	/bin/true | /bin/true | /bin/true | /bin/true | /bin/true | /bin/true | bin/true | /bin/true |
   550  	/bin/true | /bin/true | /bin/true | /bin/true | /bin/true | /bin/true | bin/true | /bin/true |
   551  	/bin/true | /bin/true | /bin/true | /bin/true | /bin/true | /bin/true | bin/true | /bin/true`)
   552  
   553  	// Enforce a restrictive limit. 64 * /bin/true + 1 * shell should cause
   554  	// this to fail reliably.
   555  	config.Cgroups.Resources.PidsLimit = 64
   556  	out, _, err := runContainer(t, config, "/bin/sh", "-c", `
   557  	/bin/true | /bin/true | /bin/true | /bin/true | /bin/true | /bin/true | bin/true | /bin/true |
   558  	/bin/true | /bin/true | /bin/true | /bin/true | /bin/true | /bin/true | bin/true | /bin/true |
   559  	/bin/true | /bin/true | /bin/true | /bin/true | /bin/true | /bin/true | bin/true | /bin/true |
   560  	/bin/true | /bin/true | /bin/true | /bin/true | /bin/true | /bin/true | bin/true | /bin/true |
   561  	/bin/true | /bin/true | /bin/true | /bin/true | /bin/true | /bin/true | bin/true | /bin/true |
   562  	/bin/true | /bin/true | /bin/true | /bin/true | /bin/true | /bin/true | bin/true | /bin/true |
   563  	/bin/true | /bin/true | /bin/true | /bin/true | /bin/true | /bin/true | bin/true | /bin/true |
   564  	/bin/true | /bin/true | /bin/true | /bin/true | /bin/true | /bin/true | bin/true | /bin/true`)
   565  	if err != nil && !strings.Contains(out.String(), "sh: can't fork") {
   566  		t.Fatal(err)
   567  	}
   568  
   569  	if err == nil {
   570  		t.Fatal("expected fork() to fail with restrictive pids limit")
   571  	}
   572  
   573  	// Minimal restrictions are not really supported, due to quirks in using Go
   574  	// due to the fact that it spawns random processes. While we do our best with
   575  	// late setting cgroup values, it's just too unreliable with very small pids.max.
   576  	// As such, we don't test that case. YMMV.
   577  }
   578  
   579  func TestCgroupResourcesUnifiedErrorOnV1(t *testing.T) {
   580  	testCgroupResourcesUnifiedErrorOnV1(t, false)
   581  }
   582  
   583  func TestCgroupResourcesUnifiedErrorOnV1Systemd(t *testing.T) {
   584  	if !systemd.IsRunningSystemd() {
   585  		t.Skip("Test requires systemd.")
   586  	}
   587  	testCgroupResourcesUnifiedErrorOnV1(t, true)
   588  }
   589  
   590  func testCgroupResourcesUnifiedErrorOnV1(t *testing.T, systemd bool) {
   591  	if testing.Short() {
   592  		return
   593  	}
   594  	if cgroups.IsCgroup2UnifiedMode() {
   595  		t.Skip("requires cgroup v1")
   596  	}
   597  
   598  	config := newTemplateConfig(t, &tParam{systemd: systemd})
   599  	config.Cgroups.Resources.Unified = map[string]string{
   600  		"memory.min": "10240",
   601  	}
   602  	_, _, err := runContainer(t, config, "true")
   603  	if !strings.Contains(err.Error(), cgroups.ErrV1NoUnified.Error()) {
   604  		t.Fatalf("expected error to contain %v, got %v", cgroups.ErrV1NoUnified, err)
   605  	}
   606  }
   607  
   608  func TestCgroupResourcesUnified(t *testing.T) {
   609  	testCgroupResourcesUnified(t, false)
   610  }
   611  
   612  func TestCgroupResourcesUnifiedSystemd(t *testing.T) {
   613  	if !systemd.IsRunningSystemd() {
   614  		t.Skip("Test requires systemd.")
   615  	}
   616  	testCgroupResourcesUnified(t, true)
   617  }
   618  
   619  func testCgroupResourcesUnified(t *testing.T, systemd bool) {
   620  	if testing.Short() {
   621  		return
   622  	}
   623  	if !cgroups.IsCgroup2UnifiedMode() {
   624  		t.Skip("requires cgroup v2")
   625  	}
   626  
   627  	config := newTemplateConfig(t, &tParam{systemd: systemd})
   628  	config.Cgroups.Resources.Memory = 536870912     // 512M
   629  	config.Cgroups.Resources.MemorySwap = 536870912 // 512M, i.e. no swap
   630  	config.Namespaces.Add(configs.NEWCGROUP, "")
   631  
   632  	testCases := []struct {
   633  		name     string
   634  		cfg      map[string]string
   635  		expError string
   636  		cmd      []string
   637  		exp      string
   638  	}{
   639  		{
   640  			name: "dummy",
   641  			cmd:  []string{"true"},
   642  			exp:  "",
   643  		},
   644  		{
   645  			name: "set memory.min",
   646  			cfg:  map[string]string{"memory.min": "131072"},
   647  			cmd:  []string{"cat", "/sys/fs/cgroup/memory.min"},
   648  			exp:  "131072\n",
   649  		},
   650  		{
   651  			name: "check memory.max",
   652  			cmd:  []string{"cat", "/sys/fs/cgroup/memory.max"},
   653  			exp:  strconv.Itoa(int(config.Cgroups.Resources.Memory)) + "\n",
   654  		},
   655  
   656  		{
   657  			name: "overwrite memory.max",
   658  			cfg:  map[string]string{"memory.max": "268435456"},
   659  			cmd:  []string{"cat", "/sys/fs/cgroup/memory.max"},
   660  			exp:  "268435456\n",
   661  		},
   662  		{
   663  			name:     "no such controller error",
   664  			cfg:      map[string]string{"privet.vsem": "vam"},
   665  			expError: "controller \"privet\" not available",
   666  		},
   667  		{
   668  			name:     "slash in key error",
   669  			cfg:      map[string]string{"bad/key": "val"},
   670  			expError: "must be a file name (no slashes)",
   671  		},
   672  		{
   673  			name:     "no dot in key error",
   674  			cfg:      map[string]string{"badkey": "val"},
   675  			expError: "must be in the form CONTROLLER.PARAMETER",
   676  		},
   677  		{
   678  			name:     "read-only parameter",
   679  			cfg:      map[string]string{"pids.current": "42"},
   680  			expError: "failed to write",
   681  		},
   682  	}
   683  
   684  	for _, tc := range testCases {
   685  		config.Cgroups.Resources.Unified = tc.cfg
   686  		buffers, ret, err := runContainer(t, config, tc.cmd...)
   687  		if tc.expError != "" {
   688  			if err == nil {
   689  				t.Errorf("case %q failed: expected error, got nil", tc.name)
   690  				continue
   691  			}
   692  			if !strings.Contains(err.Error(), tc.expError) {
   693  				t.Errorf("case %q failed: expected error to contain %q, got %q", tc.name, tc.expError, err)
   694  			}
   695  			continue
   696  		}
   697  		if err != nil {
   698  			t.Errorf("case %q failed: expected no error, got %v (command: %v, status: %d, stderr: %q)",
   699  				tc.name, err, tc.cmd, ret, buffers.Stderr.String())
   700  			continue
   701  		}
   702  		if tc.exp != "" {
   703  			out := buffers.Stdout.String()
   704  			if out != tc.exp {
   705  				t.Errorf("expected %q, got %q", tc.exp, out)
   706  			}
   707  		}
   708  	}
   709  }
   710  
   711  func TestContainerState(t *testing.T) {
   712  	if testing.Short() {
   713  		return
   714  	}
   715  
   716  	l, err := os.Readlink("/proc/1/ns/ipc")
   717  	ok(t, err)
   718  
   719  	config := newTemplateConfig(t, nil)
   720  	config.Namespaces = configs.Namespaces([]configs.Namespace{
   721  		{Type: configs.NEWNS},
   722  		{Type: configs.NEWUTS},
   723  		// host for IPC
   724  		//{Type: configs.NEWIPC},
   725  		{Type: configs.NEWPID},
   726  		{Type: configs.NEWNET},
   727  	})
   728  
   729  	container, err := newContainer(t, config)
   730  	ok(t, err)
   731  	defer destroyContainer(container)
   732  
   733  	stdinR, stdinW, err := os.Pipe()
   734  	ok(t, err)
   735  
   736  	p := &libcontainer.Process{
   737  		Cwd:   "/",
   738  		Args:  []string{"cat"},
   739  		Env:   standardEnvironment,
   740  		Stdin: stdinR,
   741  		Init:  true,
   742  	}
   743  	err = container.Run(p)
   744  	ok(t, err)
   745  	_ = stdinR.Close()
   746  	defer stdinW.Close() //nolint: errcheck
   747  
   748  	st, err := container.State()
   749  	ok(t, err)
   750  
   751  	l1, err := os.Readlink(st.NamespacePaths[configs.NEWIPC])
   752  	ok(t, err)
   753  	if l1 != l {
   754  		t.Fatal("Container using non-host ipc namespace")
   755  	}
   756  	_ = stdinW.Close()
   757  	waitProcess(p, t)
   758  }
   759  
   760  func TestPassExtraFiles(t *testing.T) {
   761  	if testing.Short() {
   762  		return
   763  	}
   764  
   765  	config := newTemplateConfig(t, nil)
   766  	container, err := newContainer(t, config)
   767  	ok(t, err)
   768  	defer destroyContainer(container)
   769  
   770  	var stdout bytes.Buffer
   771  	pipeout1, pipein1, err := os.Pipe()
   772  	ok(t, err)
   773  	pipeout2, pipein2, err := os.Pipe()
   774  	ok(t, err)
   775  	process := libcontainer.Process{
   776  		Cwd:        "/",
   777  		Args:       []string{"sh", "-c", "cd /proc/$$/fd; echo -n *; echo -n 1 >3; echo -n 2 >4"},
   778  		Env:        []string{"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"},
   779  		ExtraFiles: []*os.File{pipein1, pipein2},
   780  		Stdin:      nil,
   781  		Stdout:     &stdout,
   782  		Init:       true,
   783  	}
   784  	err = container.Run(&process)
   785  	ok(t, err)
   786  
   787  	waitProcess(&process, t)
   788  
   789  	out := stdout.String()
   790  	// fd 5 is the directory handle for /proc/$$/fd
   791  	if out != "0 1 2 3 4 5" {
   792  		t.Fatalf("expected to have the file descriptors '0 1 2 3 4 5' passed to init, got '%s'", out)
   793  	}
   794  	buf := []byte{0}
   795  	_, err = pipeout1.Read(buf)
   796  	ok(t, err)
   797  	out1 := string(buf)
   798  	if out1 != "1" {
   799  		t.Fatalf("expected first pipe to receive '1', got '%s'", out1)
   800  	}
   801  
   802  	_, err = pipeout2.Read(buf)
   803  	ok(t, err)
   804  	out2 := string(buf)
   805  	if out2 != "2" {
   806  		t.Fatalf("expected second pipe to receive '2', got '%s'", out2)
   807  	}
   808  }
   809  
   810  func TestMountCmds(t *testing.T) {
   811  	if testing.Short() {
   812  		return
   813  	}
   814  
   815  	tmpDir := t.TempDir()
   816  	config := newTemplateConfig(t, nil)
   817  	rootfs := config.Rootfs
   818  	config.Mounts = append(config.Mounts, &configs.Mount{
   819  		Source:      tmpDir,
   820  		Destination: "/tmp",
   821  		Device:      "bind",
   822  		Flags:       unix.MS_BIND | unix.MS_REC,
   823  		PremountCmds: []configs.Command{
   824  			{Path: "touch", Args: []string{filepath.Join(tmpDir, "hello")}},
   825  			{Path: "touch", Args: []string{filepath.Join(tmpDir, "world")}},
   826  		},
   827  		PostmountCmds: []configs.Command{
   828  			{Path: "cp", Args: []string{filepath.Join(rootfs, "tmp", "hello"), filepath.Join(rootfs, "tmp", "hello-backup")}},
   829  			{Path: "cp", Args: []string{filepath.Join(rootfs, "tmp", "world"), filepath.Join(rootfs, "tmp", "world-backup")}},
   830  		},
   831  	})
   832  
   833  	container, err := newContainer(t, config)
   834  	ok(t, err)
   835  	defer destroyContainer(container)
   836  
   837  	pconfig := libcontainer.Process{
   838  		Cwd:  "/",
   839  		Args: []string{"sh", "-c", "env"},
   840  		Env:  standardEnvironment,
   841  		Init: true,
   842  	}
   843  	err = container.Run(&pconfig)
   844  	ok(t, err)
   845  
   846  	// Wait for process
   847  	waitProcess(&pconfig, t)
   848  
   849  	entries, err := os.ReadDir(tmpDir)
   850  	ok(t, err)
   851  	expected := []string{"hello", "hello-backup", "world", "world-backup"}
   852  	for i, e := range entries {
   853  		if e.Name() != expected[i] {
   854  			t.Errorf("Got(%s), expect %s", e.Name(), expected[i])
   855  		}
   856  	}
   857  }
   858  
   859  func TestSysctl(t *testing.T) {
   860  	if testing.Short() {
   861  		return
   862  	}
   863  
   864  	config := newTemplateConfig(t, nil)
   865  	config.Sysctl = map[string]string{
   866  		"kernel.shmmni": "8192",
   867  		"kernel/shmmax": "4194304",
   868  	}
   869  	const (
   870  		cmd = "cat shmmni shmmax"
   871  		exp = "8192\n4194304\n"
   872  	)
   873  
   874  	container, err := newContainer(t, config)
   875  	ok(t, err)
   876  	defer destroyContainer(container)
   877  
   878  	var stdout bytes.Buffer
   879  	pconfig := libcontainer.Process{
   880  		Cwd:    "/proc/sys/kernel",
   881  		Args:   []string{"sh", "-c", cmd},
   882  		Env:    standardEnvironment,
   883  		Stdin:  nil,
   884  		Stdout: &stdout,
   885  		Init:   true,
   886  	}
   887  	err = container.Run(&pconfig)
   888  	ok(t, err)
   889  
   890  	// Wait for process
   891  	waitProcess(&pconfig, t)
   892  
   893  	out := stdout.String()
   894  	if out != exp {
   895  		t.Fatalf("expected %s, got %s", exp, out)
   896  	}
   897  }
   898  
   899  func TestMountCgroupRO(t *testing.T) {
   900  	if testing.Short() {
   901  		return
   902  	}
   903  	config := newTemplateConfig(t, nil)
   904  	buffers := runContainerOk(t, config, "mount")
   905  
   906  	mountInfo := buffers.Stdout.String()
   907  	lines := strings.Split(mountInfo, "\n")
   908  	for _, l := range lines {
   909  		if strings.HasPrefix(l, "tmpfs on /sys/fs/cgroup") {
   910  			if !strings.Contains(l, "ro") ||
   911  				!strings.Contains(l, "nosuid") ||
   912  				!strings.Contains(l, "nodev") ||
   913  				!strings.Contains(l, "noexec") {
   914  				t.Fatalf("Mode expected to contain 'ro,nosuid,nodev,noexec': %s", l)
   915  			}
   916  			if !strings.Contains(l, "mode=755") {
   917  				t.Fatalf("Mode expected to contain 'mode=755': %s", l)
   918  			}
   919  			continue
   920  		}
   921  		if !strings.HasPrefix(l, "cgroup") {
   922  			continue
   923  		}
   924  		if !strings.Contains(l, "ro") ||
   925  			!strings.Contains(l, "nosuid") ||
   926  			!strings.Contains(l, "nodev") ||
   927  			!strings.Contains(l, "noexec") {
   928  			t.Fatalf("Mode expected to contain 'ro,nosuid,nodev,noexec': %s", l)
   929  		}
   930  	}
   931  }
   932  
   933  func TestMountCgroupRW(t *testing.T) {
   934  	if testing.Short() {
   935  		return
   936  	}
   937  	config := newTemplateConfig(t, nil)
   938  	// clear the RO flag from cgroup mount
   939  	for _, m := range config.Mounts {
   940  		if m.Device == "cgroup" {
   941  			m.Flags = defaultMountFlags
   942  			break
   943  		}
   944  	}
   945  
   946  	buffers := runContainerOk(t, config, "mount")
   947  
   948  	mountInfo := buffers.Stdout.String()
   949  	lines := strings.Split(mountInfo, "\n")
   950  	for _, l := range lines {
   951  		if strings.HasPrefix(l, "tmpfs on /sys/fs/cgroup") {
   952  			if !strings.Contains(l, "rw") ||
   953  				!strings.Contains(l, "nosuid") ||
   954  				!strings.Contains(l, "nodev") ||
   955  				!strings.Contains(l, "noexec") {
   956  				t.Fatalf("Mode expected to contain 'rw,nosuid,nodev,noexec': %s", l)
   957  			}
   958  			if !strings.Contains(l, "mode=755") {
   959  				t.Fatalf("Mode expected to contain 'mode=755': %s", l)
   960  			}
   961  			continue
   962  		}
   963  		if !strings.HasPrefix(l, "cgroup") {
   964  			continue
   965  		}
   966  		if !strings.Contains(l, "rw") ||
   967  			!strings.Contains(l, "nosuid") ||
   968  			!strings.Contains(l, "nodev") ||
   969  			!strings.Contains(l, "noexec") {
   970  			t.Fatalf("Mode expected to contain 'rw,nosuid,nodev,noexec': %s", l)
   971  		}
   972  	}
   973  }
   974  
   975  func TestOomScoreAdj(t *testing.T) {
   976  	if testing.Short() {
   977  		return
   978  	}
   979  
   980  	config := newTemplateConfig(t, nil)
   981  	config.OomScoreAdj = ptrInt(200)
   982  
   983  	container, err := newContainer(t, config)
   984  	ok(t, err)
   985  	defer destroyContainer(container)
   986  
   987  	var stdout bytes.Buffer
   988  	pconfig := libcontainer.Process{
   989  		Cwd:    "/",
   990  		Args:   []string{"sh", "-c", "cat /proc/self/oom_score_adj"},
   991  		Env:    standardEnvironment,
   992  		Stdin:  nil,
   993  		Stdout: &stdout,
   994  		Init:   true,
   995  	}
   996  	err = container.Run(&pconfig)
   997  	ok(t, err)
   998  
   999  	// Wait for process
  1000  	waitProcess(&pconfig, t)
  1001  	outputOomScoreAdj := strings.TrimSpace(stdout.String())
  1002  
  1003  	// Check that the oom_score_adj matches the value that was set as part of config.
  1004  	if outputOomScoreAdj != strconv.Itoa(*config.OomScoreAdj) {
  1005  		t.Fatalf("Expected oom_score_adj %d; got %q", *config.OomScoreAdj, outputOomScoreAdj)
  1006  	}
  1007  }
  1008  
  1009  func TestHook(t *testing.T) {
  1010  	if testing.Short() {
  1011  		return
  1012  	}
  1013  
  1014  	config := newTemplateConfig(t, nil)
  1015  	expectedBundle := t.TempDir()
  1016  	config.Labels = append(config.Labels, "bundle="+expectedBundle)
  1017  
  1018  	getRootfsFromBundle := func(bundle string) (string, error) {
  1019  		f, err := os.Open(filepath.Join(bundle, "config.json"))
  1020  		if err != nil {
  1021  			return "", err
  1022  		}
  1023  
  1024  		var config configs.Config
  1025  		if err = json.NewDecoder(f).Decode(&config); err != nil {
  1026  			return "", err
  1027  		}
  1028  		return config.Rootfs, nil
  1029  	}
  1030  	createFileFromBundle := func(filename, bundle string) error {
  1031  		root, err := getRootfsFromBundle(bundle)
  1032  		if err != nil {
  1033  			return err
  1034  		}
  1035  
  1036  		f, err := os.Create(filepath.Join(root, filename))
  1037  		if err != nil {
  1038  			return err
  1039  		}
  1040  		return f.Close()
  1041  	}
  1042  
  1043  	// Note FunctionHooks can't be serialized to json this means they won't be passed down to the container
  1044  	// For CreateContainer and StartContainer which run in the container namespace, this means we need to pass Command Hooks.
  1045  	hookFiles := map[configs.HookName]string{
  1046  		configs.Prestart:        "prestart",
  1047  		configs.CreateRuntime:   "createRuntime",
  1048  		configs.CreateContainer: "createContainer",
  1049  		configs.StartContainer:  "startContainer",
  1050  		configs.Poststart:       "poststart",
  1051  	}
  1052  
  1053  	config.Hooks = configs.Hooks{
  1054  		configs.Prestart: configs.HookList{
  1055  			configs.NewFunctionHook(func(s *specs.State) error {
  1056  				if s.Bundle != expectedBundle {
  1057  					t.Fatalf("Expected prestart hook bundlePath '%s'; got '%s'", expectedBundle, s.Bundle)
  1058  				}
  1059  				return createFileFromBundle(hookFiles[configs.Prestart], s.Bundle)
  1060  			}),
  1061  		},
  1062  		configs.CreateRuntime: configs.HookList{
  1063  			configs.NewFunctionHook(func(s *specs.State) error {
  1064  				if s.Bundle != expectedBundle {
  1065  					t.Fatalf("Expected createRuntime hook bundlePath '%s'; got '%s'", expectedBundle, s.Bundle)
  1066  				}
  1067  				return createFileFromBundle(hookFiles[configs.CreateRuntime], s.Bundle)
  1068  			}),
  1069  		},
  1070  		configs.CreateContainer: configs.HookList{
  1071  			configs.NewCommandHook(configs.Command{
  1072  				Path: "/bin/bash",
  1073  				Args: []string{"/bin/bash", "-c", fmt.Sprintf("touch ./%s", hookFiles[configs.CreateContainer])},
  1074  			}),
  1075  		},
  1076  		configs.StartContainer: configs.HookList{
  1077  			configs.NewCommandHook(configs.Command{
  1078  				Path: "/bin/sh",
  1079  				Args: []string{"/bin/sh", "-c", fmt.Sprintf("touch /%s", hookFiles[configs.StartContainer])},
  1080  			}),
  1081  		},
  1082  		configs.Poststart: configs.HookList{
  1083  			configs.NewFunctionHook(func(s *specs.State) error {
  1084  				if s.Bundle != expectedBundle {
  1085  					t.Fatalf("Expected poststart hook bundlePath '%s'; got '%s'", expectedBundle, s.Bundle)
  1086  				}
  1087  				return createFileFromBundle(hookFiles[configs.Poststart], s.Bundle)
  1088  			}),
  1089  		},
  1090  		configs.Poststop: configs.HookList{
  1091  			configs.NewFunctionHook(func(s *specs.State) error {
  1092  				if s.Bundle != expectedBundle {
  1093  					t.Fatalf("Expected poststop hook bundlePath '%s'; got '%s'", expectedBundle, s.Bundle)
  1094  				}
  1095  
  1096  				root, err := getRootfsFromBundle(s.Bundle)
  1097  				if err != nil {
  1098  					return err
  1099  				}
  1100  
  1101  				for _, hook := range hookFiles {
  1102  					if err = os.RemoveAll(filepath.Join(root, hook)); err != nil {
  1103  						return err
  1104  					}
  1105  				}
  1106  				return nil
  1107  			}),
  1108  		},
  1109  	}
  1110  
  1111  	// write config of json format into config.json under bundle
  1112  	f, err := os.OpenFile(filepath.Join(expectedBundle, "config.json"), os.O_CREATE|os.O_RDWR, 0o644)
  1113  	ok(t, err)
  1114  	ok(t, json.NewEncoder(f).Encode(config))
  1115  
  1116  	container, err := newContainer(t, config)
  1117  	ok(t, err)
  1118  
  1119  	// e.g: 'ls /prestart ...'
  1120  	cmd := "ls "
  1121  	for _, hook := range hookFiles {
  1122  		cmd += "/" + hook + " "
  1123  	}
  1124  
  1125  	var stdout bytes.Buffer
  1126  	pconfig := libcontainer.Process{
  1127  		Cwd:    "/",
  1128  		Args:   []string{"sh", "-c", cmd},
  1129  		Env:    standardEnvironment,
  1130  		Stdin:  nil,
  1131  		Stdout: &stdout,
  1132  		Init:   true,
  1133  	}
  1134  	err = container.Run(&pconfig)
  1135  	ok(t, err)
  1136  
  1137  	// Wait for process
  1138  	waitProcess(&pconfig, t)
  1139  
  1140  	if err := container.Destroy(); err != nil {
  1141  		t.Fatalf("container destroy %s", err)
  1142  	}
  1143  
  1144  	for _, hook := range []string{"prestart", "createRuntime", "poststart"} {
  1145  		fi, err := os.Stat(filepath.Join(config.Rootfs, hook))
  1146  		if err == nil || !os.IsNotExist(err) {
  1147  			t.Fatalf("expected file '%s to not exists, but it does", fi.Name())
  1148  		}
  1149  	}
  1150  }
  1151  
  1152  func TestSTDIOPermissions(t *testing.T) {
  1153  	if testing.Short() {
  1154  		return
  1155  	}
  1156  
  1157  	config := newTemplateConfig(t, nil)
  1158  	buffers := runContainerOk(t, config, "sh", "-c", "echo hi > /dev/stderr")
  1159  
  1160  	if actual := strings.Trim(buffers.Stderr.String(), "\n"); actual != "hi" {
  1161  		t.Fatalf("stderr should equal be equal %q %q", actual, "hi")
  1162  	}
  1163  }
  1164  
  1165  func unmountOp(path string) {
  1166  	_ = unix.Unmount(path, unix.MNT_DETACH)
  1167  }
  1168  
  1169  // Launch container with rootfsPropagation in rslave mode. Also
  1170  // bind mount a volume /mnt1host at /mnt1cont at the time of launch. Now do
  1171  // another mount on host (/mnt1host/mnt2host) and this new mount should
  1172  // propagate to container (/mnt1cont/mnt2host)
  1173  func TestRootfsPropagationSlaveMount(t *testing.T) {
  1174  	var mountPropagated bool
  1175  	var dir1cont string
  1176  	var dir2cont string
  1177  
  1178  	dir1cont = "/root/mnt1cont"
  1179  
  1180  	if testing.Short() {
  1181  		return
  1182  	}
  1183  	config := newTemplateConfig(t, nil)
  1184  	config.RootPropagation = unix.MS_SLAVE | unix.MS_REC
  1185  
  1186  	// Bind mount a volume.
  1187  	dir1host := t.TempDir()
  1188  
  1189  	// Make this dir a "shared" mount point. This will make sure a
  1190  	// slave relationship can be established in container.
  1191  	err := unix.Mount(dir1host, dir1host, "bind", unix.MS_BIND|unix.MS_REC, "")
  1192  	ok(t, err)
  1193  	err = unix.Mount("", dir1host, "", unix.MS_SHARED|unix.MS_REC, "")
  1194  	ok(t, err)
  1195  	defer unmountOp(dir1host)
  1196  
  1197  	config.Mounts = append(config.Mounts, &configs.Mount{
  1198  		Source:      dir1host,
  1199  		Destination: dir1cont,
  1200  		Device:      "bind",
  1201  		Flags:       unix.MS_BIND | unix.MS_REC,
  1202  	})
  1203  
  1204  	container, err := newContainer(t, config)
  1205  	ok(t, err)
  1206  	defer destroyContainer(container)
  1207  
  1208  	stdinR, stdinW, err := os.Pipe()
  1209  	ok(t, err)
  1210  
  1211  	pconfig := &libcontainer.Process{
  1212  		Cwd:   "/",
  1213  		Args:  []string{"cat"},
  1214  		Env:   standardEnvironment,
  1215  		Stdin: stdinR,
  1216  		Init:  true,
  1217  	}
  1218  
  1219  	err = container.Run(pconfig)
  1220  	_ = stdinR.Close()
  1221  	defer stdinW.Close() //nolint: errcheck
  1222  	ok(t, err)
  1223  
  1224  	// Create mnt2host under dir1host and bind mount itself on top of it.
  1225  	// This should be visible in container.
  1226  	dir2host := filepath.Join(dir1host, "mnt2host")
  1227  	err = os.Mkdir(dir2host, 0o700)
  1228  	ok(t, err)
  1229  	defer remove(dir2host)
  1230  
  1231  	err = unix.Mount(dir2host, dir2host, "bind", unix.MS_BIND, "")
  1232  	defer unmountOp(dir2host)
  1233  	ok(t, err)
  1234  
  1235  	// Run "cat /proc/self/mountinfo" in container and look at mount points.
  1236  	var stdout2 bytes.Buffer
  1237  
  1238  	stdinR2, stdinW2, err := os.Pipe()
  1239  	ok(t, err)
  1240  
  1241  	pconfig2 := &libcontainer.Process{
  1242  		Cwd:    "/",
  1243  		Args:   []string{"cat", "/proc/self/mountinfo"},
  1244  		Env:    standardEnvironment,
  1245  		Stdin:  stdinR2,
  1246  		Stdout: &stdout2,
  1247  	}
  1248  
  1249  	err = container.Run(pconfig2)
  1250  	_ = stdinR2.Close()
  1251  	defer stdinW2.Close() //nolint: errcheck
  1252  	ok(t, err)
  1253  
  1254  	_ = stdinW2.Close()
  1255  	waitProcess(pconfig2, t)
  1256  	_ = stdinW.Close()
  1257  	waitProcess(pconfig, t)
  1258  
  1259  	mountPropagated = false
  1260  	dir2cont = filepath.Join(dir1cont, filepath.Base(dir2host))
  1261  
  1262  	propagationInfo := stdout2.String()
  1263  	lines := strings.Split(propagationInfo, "\n")
  1264  	for _, l := range lines {
  1265  		linefields := strings.Split(l, " ")
  1266  		if len(linefields) < 5 {
  1267  			continue
  1268  		}
  1269  
  1270  		if linefields[4] == dir2cont {
  1271  			mountPropagated = true
  1272  			break
  1273  		}
  1274  	}
  1275  
  1276  	if mountPropagated != true {
  1277  		t.Fatalf("Mount on host %s did not propagate in container at %s\n", dir2host, dir2cont)
  1278  	}
  1279  }
  1280  
  1281  // Launch container with rootfsPropagation 0 so no propagation flags are
  1282  // applied. Also bind mount a volume /mnt1host at /mnt1cont at the time of
  1283  // launch. Now do a mount in container (/mnt1cont/mnt2cont) and this new
  1284  // mount should propagate to host (/mnt1host/mnt2cont)
  1285  
  1286  func TestRootfsPropagationSharedMount(t *testing.T) {
  1287  	var dir1cont string
  1288  	var dir2cont string
  1289  
  1290  	dir1cont = "/root/mnt1cont"
  1291  
  1292  	if testing.Short() {
  1293  		return
  1294  	}
  1295  	config := newTemplateConfig(t, nil)
  1296  	config.RootPropagation = unix.MS_PRIVATE
  1297  
  1298  	// Bind mount a volume.
  1299  	dir1host := t.TempDir()
  1300  
  1301  	// Make this dir a "shared" mount point. This will make sure a
  1302  	// shared relationship can be established in container.
  1303  	err := unix.Mount(dir1host, dir1host, "bind", unix.MS_BIND|unix.MS_REC, "")
  1304  	ok(t, err)
  1305  	err = unix.Mount("", dir1host, "", unix.MS_SHARED|unix.MS_REC, "")
  1306  	ok(t, err)
  1307  	defer unmountOp(dir1host)
  1308  
  1309  	config.Mounts = append(config.Mounts, &configs.Mount{
  1310  		Source:      dir1host,
  1311  		Destination: dir1cont,
  1312  		Device:      "bind",
  1313  		Flags:       unix.MS_BIND | unix.MS_REC,
  1314  	})
  1315  
  1316  	container, err := newContainer(t, config)
  1317  	ok(t, err)
  1318  	defer destroyContainer(container)
  1319  
  1320  	stdinR, stdinW, err := os.Pipe()
  1321  	ok(t, err)
  1322  
  1323  	pconfig := &libcontainer.Process{
  1324  		Cwd:   "/",
  1325  		Args:  []string{"cat"},
  1326  		Env:   standardEnvironment,
  1327  		Stdin: stdinR,
  1328  		Init:  true,
  1329  	}
  1330  
  1331  	err = container.Run(pconfig)
  1332  	_ = stdinR.Close()
  1333  	defer stdinW.Close() //nolint: errcheck
  1334  	ok(t, err)
  1335  
  1336  	// Create mnt2cont under dir1host. This will become visible inside container
  1337  	// at mnt1cont/mnt2cont. Bind mount itself on top of it. This
  1338  	// should be visible on host now.
  1339  	dir2host := filepath.Join(dir1host, "mnt2cont")
  1340  	err = os.Mkdir(dir2host, 0o700)
  1341  	ok(t, err)
  1342  	defer remove(dir2host)
  1343  
  1344  	dir2cont = filepath.Join(dir1cont, filepath.Base(dir2host))
  1345  
  1346  	// Mount something in container and see if it is visible on host.
  1347  	var stdout2 bytes.Buffer
  1348  
  1349  	stdinR2, stdinW2, err := os.Pipe()
  1350  	ok(t, err)
  1351  
  1352  	pconfig2 := &libcontainer.Process{
  1353  		Cwd:          "/",
  1354  		Args:         []string{"mount", "--bind", dir2cont, dir2cont},
  1355  		Env:          standardEnvironment,
  1356  		Stdin:        stdinR2,
  1357  		Stdout:       &stdout2,
  1358  		Capabilities: &configs.Capabilities{},
  1359  	}
  1360  
  1361  	// Provide CAP_SYS_ADMIN
  1362  	pconfig2.Capabilities.Bounding = append(config.Capabilities.Bounding, "CAP_SYS_ADMIN")
  1363  	pconfig2.Capabilities.Permitted = append(config.Capabilities.Permitted, "CAP_SYS_ADMIN")
  1364  	pconfig2.Capabilities.Effective = append(config.Capabilities.Effective, "CAP_SYS_ADMIN")
  1365  
  1366  	err = container.Run(pconfig2)
  1367  	_ = stdinR2.Close()
  1368  	defer stdinW2.Close() //nolint: errcheck
  1369  	ok(t, err)
  1370  
  1371  	// Wait for process
  1372  	_ = stdinW2.Close()
  1373  	waitProcess(pconfig2, t)
  1374  	_ = stdinW.Close()
  1375  	waitProcess(pconfig, t)
  1376  
  1377  	defer unmountOp(dir2host)
  1378  
  1379  	// Check if mount is visible on host or not.
  1380  	out, err := exec.Command("findmnt", "-n", "-f", "-oTARGET", dir2host).CombinedOutput()
  1381  	outtrim := string(bytes.TrimSpace(out))
  1382  	if err != nil {
  1383  		t.Logf("findmnt error %q: %q", err, outtrim)
  1384  	}
  1385  
  1386  	if outtrim != dir2host {
  1387  		t.Fatalf("Mount in container on %s did not propagate to host on %s. finmnt output=%s", dir2cont, dir2host, outtrim)
  1388  	}
  1389  }
  1390  
  1391  func TestPIDHost(t *testing.T) {
  1392  	if testing.Short() {
  1393  		return
  1394  	}
  1395  
  1396  	l, err := os.Readlink("/proc/1/ns/pid")
  1397  	ok(t, err)
  1398  
  1399  	config := newTemplateConfig(t, nil)
  1400  	config.Namespaces.Remove(configs.NEWPID)
  1401  	buffers := runContainerOk(t, config, "readlink", "/proc/self/ns/pid")
  1402  
  1403  	if actual := strings.Trim(buffers.Stdout.String(), "\n"); actual != l {
  1404  		t.Fatalf("ipc link not equal to host link %q %q", actual, l)
  1405  	}
  1406  }
  1407  
  1408  func TestPIDHostInitProcessWait(t *testing.T) {
  1409  	if testing.Short() {
  1410  		return
  1411  	}
  1412  
  1413  	pidns := "/proc/1/ns/pid"
  1414  
  1415  	// Run a container with two long-running processes.
  1416  	config := newTemplateConfig(t, nil)
  1417  	config.Namespaces.Add(configs.NEWPID, pidns)
  1418  	container, err := newContainer(t, config)
  1419  	ok(t, err)
  1420  	defer func() {
  1421  		_ = container.Destroy()
  1422  	}()
  1423  
  1424  	process1 := &libcontainer.Process{
  1425  		Cwd:  "/",
  1426  		Args: []string{"sleep", "100"},
  1427  		Env:  standardEnvironment,
  1428  		Init: true,
  1429  	}
  1430  	err = container.Run(process1)
  1431  	ok(t, err)
  1432  
  1433  	process2 := &libcontainer.Process{
  1434  		Cwd:  "/",
  1435  		Args: []string{"sleep", "100"},
  1436  		Env:  standardEnvironment,
  1437  		Init: false,
  1438  	}
  1439  	err = container.Run(process2)
  1440  	ok(t, err)
  1441  
  1442  	// Kill the init process and Wait for it.
  1443  	err = process1.Signal(syscall.SIGKILL)
  1444  	ok(t, err)
  1445  	_, err = process1.Wait()
  1446  	if err == nil {
  1447  		t.Fatal("expected Wait to indicate failure")
  1448  	}
  1449  
  1450  	// The non-init process must've been killed.
  1451  	err = process2.Signal(syscall.Signal(0))
  1452  	if err == nil || err.Error() != "no such process" {
  1453  		t.Fatalf("expected process to have been killed: %v", err)
  1454  	}
  1455  }
  1456  
  1457  func TestInitJoinPID(t *testing.T) {
  1458  	if testing.Short() {
  1459  		return
  1460  	}
  1461  	// Execute a long-running container
  1462  	config1 := newTemplateConfig(t, nil)
  1463  	container1, err := newContainer(t, config1)
  1464  	ok(t, err)
  1465  	defer destroyContainer(container1)
  1466  
  1467  	stdinR1, stdinW1, err := os.Pipe()
  1468  	ok(t, err)
  1469  	init1 := &libcontainer.Process{
  1470  		Cwd:   "/",
  1471  		Args:  []string{"cat"},
  1472  		Env:   standardEnvironment,
  1473  		Stdin: stdinR1,
  1474  		Init:  true,
  1475  	}
  1476  	err = container1.Run(init1)
  1477  	_ = stdinR1.Close()
  1478  	defer stdinW1.Close() //nolint: errcheck
  1479  	ok(t, err)
  1480  
  1481  	// get the state of the first container
  1482  	state1, err := container1.State()
  1483  	ok(t, err)
  1484  	pidns1 := state1.NamespacePaths[configs.NEWPID]
  1485  
  1486  	// Run a container inside the existing pidns but with different cgroups
  1487  	config2 := newTemplateConfig(t, nil)
  1488  	config2.Namespaces.Add(configs.NEWPID, pidns1)
  1489  	config2.Cgroups.Path = "integration/test2"
  1490  	container2, err := newContainer(t, config2)
  1491  	ok(t, err)
  1492  	defer destroyContainer(container2)
  1493  
  1494  	stdinR2, stdinW2, err := os.Pipe()
  1495  	ok(t, err)
  1496  	init2 := &libcontainer.Process{
  1497  		Cwd:   "/",
  1498  		Args:  []string{"cat"},
  1499  		Env:   standardEnvironment,
  1500  		Stdin: stdinR2,
  1501  		Init:  true,
  1502  	}
  1503  	err = container2.Run(init2)
  1504  	_ = stdinR2.Close()
  1505  	defer stdinW2.Close() //nolint: errcheck
  1506  	ok(t, err)
  1507  	// get the state of the second container
  1508  	state2, err := container2.State()
  1509  	ok(t, err)
  1510  
  1511  	ns1, err := os.Readlink(fmt.Sprintf("/proc/%d/ns/pid", state1.InitProcessPid))
  1512  	ok(t, err)
  1513  	ns2, err := os.Readlink(fmt.Sprintf("/proc/%d/ns/pid", state2.InitProcessPid))
  1514  	ok(t, err)
  1515  	if ns1 != ns2 {
  1516  		t.Errorf("pidns(%s), wanted %s", ns2, ns1)
  1517  	}
  1518  
  1519  	// check that namespaces are not the same
  1520  	if reflect.DeepEqual(state2.NamespacePaths, state1.NamespacePaths) {
  1521  		t.Errorf("Namespaces(%v), original %v", state2.NamespacePaths,
  1522  			state1.NamespacePaths)
  1523  	}
  1524  	// check that pidns is joined correctly. The initial container process list
  1525  	// should contain the second container's init process
  1526  	buffers := newStdBuffers()
  1527  	ps := &libcontainer.Process{
  1528  		Cwd:    "/",
  1529  		Args:   []string{"ps"},
  1530  		Env:    standardEnvironment,
  1531  		Stdout: buffers.Stdout,
  1532  	}
  1533  	err = container1.Run(ps)
  1534  	ok(t, err)
  1535  	waitProcess(ps, t)
  1536  
  1537  	// Stop init processes one by one. Stop the second container should
  1538  	// not stop the first.
  1539  	_ = stdinW2.Close()
  1540  	waitProcess(init2, t)
  1541  	_ = stdinW1.Close()
  1542  	waitProcess(init1, t)
  1543  
  1544  	out := strings.TrimSpace(buffers.Stdout.String())
  1545  	// output of ps inside the initial PID namespace should have
  1546  	// 1 line of header,
  1547  	// 2 lines of init processes,
  1548  	// 1 line of ps process
  1549  	if len(strings.Split(out, "\n")) != 4 {
  1550  		t.Errorf("unexpected running process, output %q", out)
  1551  	}
  1552  }
  1553  
  1554  func TestInitJoinNetworkAndUser(t *testing.T) {
  1555  	if _, err := os.Stat("/proc/self/ns/user"); os.IsNotExist(err) {
  1556  		t.Skip("Test requires userns.")
  1557  	}
  1558  	if testing.Short() {
  1559  		return
  1560  	}
  1561  
  1562  	// Execute a long-running container
  1563  	config1 := newTemplateConfig(t, &tParam{userns: true})
  1564  	container1, err := newContainer(t, config1)
  1565  	ok(t, err)
  1566  	defer destroyContainer(container1)
  1567  
  1568  	stdinR1, stdinW1, err := os.Pipe()
  1569  	ok(t, err)
  1570  	init1 := &libcontainer.Process{
  1571  		Cwd:   "/",
  1572  		Args:  []string{"cat"},
  1573  		Env:   standardEnvironment,
  1574  		Stdin: stdinR1,
  1575  		Init:  true,
  1576  	}
  1577  	err = container1.Run(init1)
  1578  	_ = stdinR1.Close()
  1579  	defer stdinW1.Close() //nolint: errcheck
  1580  	ok(t, err)
  1581  
  1582  	// get the state of the first container
  1583  	state1, err := container1.State()
  1584  	ok(t, err)
  1585  	netns1 := state1.NamespacePaths[configs.NEWNET]
  1586  	userns1 := state1.NamespacePaths[configs.NEWUSER]
  1587  
  1588  	// Run a container inside the existing pidns but with different cgroups.
  1589  	config2 := newTemplateConfig(t, &tParam{userns: true})
  1590  	config2.Namespaces.Add(configs.NEWNET, netns1)
  1591  	config2.Namespaces.Add(configs.NEWUSER, userns1)
  1592  	// Emulate specconv.setupUserNamespace().
  1593  	uidMap, gidMap, err := userns.GetUserNamespaceMappings(userns1)
  1594  	ok(t, err)
  1595  	config2.UidMappings = uidMap
  1596  	config2.GidMappings = gidMap
  1597  	config2.Cgroups.Path = "integration/test2"
  1598  	container2, err := newContainer(t, config2)
  1599  	ok(t, err)
  1600  	defer destroyContainer(container2)
  1601  
  1602  	stdinR2, stdinW2, err := os.Pipe()
  1603  	ok(t, err)
  1604  	init2 := &libcontainer.Process{
  1605  		Cwd:   "/",
  1606  		Args:  []string{"cat"},
  1607  		Env:   standardEnvironment,
  1608  		Stdin: stdinR2,
  1609  		Init:  true,
  1610  	}
  1611  	err = container2.Run(init2)
  1612  	_ = stdinR2.Close()
  1613  	defer stdinW2.Close() //nolint: errcheck
  1614  	ok(t, err)
  1615  
  1616  	// get the state of the second container
  1617  	state2, err := container2.State()
  1618  	ok(t, err)
  1619  
  1620  	for _, ns := range []string{"net", "user"} {
  1621  		ns1, err := os.Readlink(fmt.Sprintf("/proc/%d/ns/%s", state1.InitProcessPid, ns))
  1622  		ok(t, err)
  1623  		ns2, err := os.Readlink(fmt.Sprintf("/proc/%d/ns/%s", state2.InitProcessPid, ns))
  1624  		ok(t, err)
  1625  		if ns1 != ns2 {
  1626  			t.Errorf("%s(%s), wanted %s", ns, ns2, ns1)
  1627  		}
  1628  	}
  1629  
  1630  	// check that namespaces are not the same
  1631  	if reflect.DeepEqual(state2.NamespacePaths, state1.NamespacePaths) {
  1632  		t.Errorf("Namespaces(%v), original %v", state2.NamespacePaths,
  1633  			state1.NamespacePaths)
  1634  	}
  1635  	// Stop init processes one by one. Stop the second container should
  1636  	// not stop the first.
  1637  	_ = stdinW2.Close()
  1638  	waitProcess(init2, t)
  1639  	_ = stdinW1.Close()
  1640  	waitProcess(init1, t)
  1641  }
  1642  
  1643  func TestTmpfsCopyUp(t *testing.T) {
  1644  	if testing.Short() {
  1645  		return
  1646  	}
  1647  
  1648  	config := newTemplateConfig(t, nil)
  1649  	config.Mounts = append(config.Mounts, &configs.Mount{
  1650  		Source:      "tmpfs",
  1651  		Destination: "/etc",
  1652  		Device:      "tmpfs",
  1653  		Extensions:  configs.EXT_COPYUP,
  1654  	})
  1655  
  1656  	container, err := newContainer(t, config)
  1657  	ok(t, err)
  1658  	defer destroyContainer(container)
  1659  
  1660  	var stdout bytes.Buffer
  1661  	pconfig := libcontainer.Process{
  1662  		Args:   []string{"ls", "/etc/passwd"},
  1663  		Env:    standardEnvironment,
  1664  		Stdin:  nil,
  1665  		Stdout: &stdout,
  1666  		Init:   true,
  1667  	}
  1668  	err = container.Run(&pconfig)
  1669  	ok(t, err)
  1670  
  1671  	// Wait for process
  1672  	waitProcess(&pconfig, t)
  1673  
  1674  	outputLs := stdout.String()
  1675  
  1676  	// Check that the ls output has /etc/passwd
  1677  	if !strings.Contains(outputLs, "/etc/passwd") {
  1678  		t.Fatalf("/etc/passwd not copied up as expected: %v", outputLs)
  1679  	}
  1680  }
  1681  
  1682  func TestCGROUPPrivate(t *testing.T) {
  1683  	if _, err := os.Stat("/proc/self/ns/cgroup"); os.IsNotExist(err) {
  1684  		t.Skip("Test requires cgroupns.")
  1685  	}
  1686  	if testing.Short() {
  1687  		return
  1688  	}
  1689  
  1690  	l, err := os.Readlink("/proc/1/ns/cgroup")
  1691  	ok(t, err)
  1692  
  1693  	config := newTemplateConfig(t, nil)
  1694  	config.Namespaces.Add(configs.NEWCGROUP, "")
  1695  	buffers := runContainerOk(t, config, "readlink", "/proc/self/ns/cgroup")
  1696  
  1697  	if actual := strings.Trim(buffers.Stdout.String(), "\n"); actual == l {
  1698  		t.Fatalf("cgroup link should be private to the container but equals host %q %q", actual, l)
  1699  	}
  1700  }
  1701  
  1702  func TestCGROUPHost(t *testing.T) {
  1703  	if _, err := os.Stat("/proc/self/ns/cgroup"); os.IsNotExist(err) {
  1704  		t.Skip("Test requires cgroupns.")
  1705  	}
  1706  	if testing.Short() {
  1707  		return
  1708  	}
  1709  
  1710  	l, err := os.Readlink("/proc/1/ns/cgroup")
  1711  	ok(t, err)
  1712  
  1713  	config := newTemplateConfig(t, nil)
  1714  	buffers := runContainerOk(t, config, "readlink", "/proc/self/ns/cgroup")
  1715  
  1716  	if actual := strings.Trim(buffers.Stdout.String(), "\n"); actual != l {
  1717  		t.Fatalf("cgroup link not equal to host link %q %q", actual, l)
  1718  	}
  1719  }
  1720  
  1721  func TestFdLeaks(t *testing.T) {
  1722  	testFdLeaks(t, false)
  1723  }
  1724  
  1725  func TestFdLeaksSystemd(t *testing.T) {
  1726  	if !systemd.IsRunningSystemd() {
  1727  		t.Skip("Test requires systemd.")
  1728  	}
  1729  	testFdLeaks(t, true)
  1730  }
  1731  
  1732  func testFdLeaks(t *testing.T, systemd bool) {
  1733  	if testing.Short() {
  1734  		return
  1735  	}
  1736  
  1737  	config := newTemplateConfig(t, &tParam{systemd: systemd})
  1738  	// Run a container once to exclude file descriptors that are only
  1739  	// opened once during the process lifetime by the library and are
  1740  	// never closed. Those are not considered leaks.
  1741  	//
  1742  	// Examples of this open-once file descriptors are:
  1743  	//  - /sys/fs/cgroup dirfd opened by prepareOpenat2 in libct/cgroups;
  1744  	//  - dbus connection opened by getConnection in libct/cgroups/systemd.
  1745  	_ = runContainerOk(t, config, "true")
  1746  
  1747  	pfd, err := os.Open("/proc/self/fd")
  1748  	ok(t, err)
  1749  	defer pfd.Close()
  1750  	fds0, err := pfd.Readdirnames(0)
  1751  	ok(t, err)
  1752  	_, err = pfd.Seek(0, 0)
  1753  	ok(t, err)
  1754  
  1755  	_ = runContainerOk(t, config, "true")
  1756  
  1757  	fds1, err := pfd.Readdirnames(0)
  1758  	ok(t, err)
  1759  
  1760  	if len(fds1) == len(fds0) {
  1761  		return
  1762  	}
  1763  	// Show the extra opened files.
  1764  
  1765  	excludedPaths := []string{
  1766  		"anon_inode:bpf-prog", // FIXME: see https://github.com/opencontainers/runc/issues/2366#issuecomment-776411392
  1767  	}
  1768  
  1769  	count := 0
  1770  next_fd:
  1771  	for _, fd1 := range fds1 {
  1772  		for _, fd0 := range fds0 {
  1773  			if fd0 == fd1 {
  1774  				continue next_fd
  1775  			}
  1776  		}
  1777  		dst, _ := os.Readlink("/proc/self/fd/" + fd1)
  1778  		for _, ex := range excludedPaths {
  1779  			if ex == dst {
  1780  				continue next_fd
  1781  			}
  1782  		}
  1783  
  1784  		count++
  1785  		t.Logf("extra fd %s -> %s", fd1, dst)
  1786  	}
  1787  	if count > 0 {
  1788  		t.Fatalf("found %d extra fds after container.Run", count)
  1789  	}
  1790  }
  1791  
  1792  // Test that a container using user namespaces is able to bind mount a folder
  1793  // that does not have permissions for group/others.
  1794  func TestBindMountAndUser(t *testing.T) {
  1795  	if _, err := os.Stat("/proc/self/ns/user"); errors.Is(err, os.ErrNotExist) {
  1796  		t.Skip("userns is unsupported")
  1797  	}
  1798  
  1799  	if testing.Short() {
  1800  		return
  1801  	}
  1802  
  1803  	temphost := t.TempDir()
  1804  	dirhost := filepath.Join(temphost, "inaccessible", "dir")
  1805  
  1806  	err := os.MkdirAll(dirhost, 0o755)
  1807  	ok(t, err)
  1808  
  1809  	err = os.WriteFile(filepath.Join(dirhost, "foo.txt"), []byte("Hello"), 0o755)
  1810  	ok(t, err)
  1811  
  1812  	// Make this dir inaccessible to "group,others".
  1813  	err = os.Chmod(filepath.Join(temphost, "inaccessible"), 0o700)
  1814  	ok(t, err)
  1815  
  1816  	config := newTemplateConfig(t, &tParam{
  1817  		userns: true,
  1818  	})
  1819  
  1820  	// Set HostID to 1000 to avoid DAC_OVERRIDE bypassing the purpose of this test.
  1821  	config.UidMappings[0].HostID = 1000
  1822  	config.GidMappings[0].HostID = 1000
  1823  
  1824  	// Set the owner of rootfs to the effective IDs in the host to avoid errors
  1825  	// while creating the folders to perform the mounts.
  1826  	err = os.Chown(config.Rootfs, 1000, 1000)
  1827  	ok(t, err)
  1828  
  1829  	config.Mounts = append(config.Mounts, &configs.Mount{
  1830  		Source:      dirhost,
  1831  		Destination: "/tmp/mnt1cont",
  1832  		Device:      "bind",
  1833  		Flags:       unix.MS_BIND | unix.MS_REC,
  1834  	})
  1835  
  1836  	container, err := newContainer(t, config)
  1837  	ok(t, err)
  1838  	defer container.Destroy() //nolint: errcheck
  1839  
  1840  	var stdout bytes.Buffer
  1841  
  1842  	pconfig := libcontainer.Process{
  1843  		Cwd:    "/",
  1844  		Args:   []string{"sh", "-c", "stat /tmp/mnt1cont/foo.txt"},
  1845  		Env:    standardEnvironment,
  1846  		Stdout: &stdout,
  1847  		Init:   true,
  1848  	}
  1849  	err = container.Run(&pconfig)
  1850  	ok(t, err)
  1851  
  1852  	waitProcess(&pconfig, t)
  1853  }
  1854  

View as plain text