1#!/bin/bash
2
3bats_require_minimum_version 1.5.0
4
5# Root directory of integration tests.
6INTEGRATION_ROOT=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
7
8# Download images, get *_IMAGE variables.
9IMAGES=$("${INTEGRATION_ROOT}"/get-images.sh)
10eval "$IMAGES"
11unset IMAGES
12
13: "${RUNC:="${INTEGRATION_ROOT}/../../runc"}"
14RECVTTY="${INTEGRATION_ROOT}/../../contrib/cmd/recvtty/recvtty"
15SD_HELPER="${INTEGRATION_ROOT}/../../contrib/cmd/sd-helper/sd-helper"
16SECCOMP_AGENT="${INTEGRATION_ROOT}/../../contrib/cmd/seccompagent/seccompagent"
17
18# Test data path.
19# shellcheck disable=SC2034
20TESTDATA="${INTEGRATION_ROOT}/testdata"
21
22# Kernel version
23KERNEL_VERSION="$(uname -r)"
24KERNEL_MAJOR="${KERNEL_VERSION%%.*}"
25KERNEL_MINOR="${KERNEL_VERSION#"$KERNEL_MAJOR".}"
26KERNEL_MINOR="${KERNEL_MINOR%%.*}"
27
28ARCH=$(uname -m)
29
30# Seccomp agent socket.
31SECCCOMP_AGENT_SOCKET="$BATS_TMPDIR/seccomp-agent.sock"
32
33# Check if we're in rootless mode.
34ROOTLESS=$(id -u)
35
36# Wrapper for runc.
37function runc() {
38 run __runc "$@"
39
40 # Some debug information to make life easier. bats will only print it if the
41 # test failed, in which case the output is useful.
42 # shellcheck disable=SC2154
43 echo "$(basename "$RUNC") $* (status=$status):" >&2
44 # shellcheck disable=SC2154
45 echo "$output" >&2
46}
47
48# Raw wrapper for runc.
49function __runc() {
50 "$RUNC" ${RUNC_USE_SYSTEMD+--systemd-cgroup} --root "$ROOT/state" "$@"
51}
52
53# Wrapper for runc spec.
54function runc_spec() {
55 local args=()
56 if [ "$ROOTLESS" -ne 0 ]; then
57 args+=("--rootless")
58 fi
59
60 runc spec "${args[@]}"
61
62 # Always add additional mappings if we have idmaps.
63 if [[ "$ROOTLESS" -ne 0 ]] && [[ "$ROOTLESS_FEATURES" == *"idmap"* ]]; then
64 runc_rootless_idmap
65 fi
66}
67
68# Helper function to reformat config.json file. Input uses jq syntax.
69function update_config() {
70 jq "$@" "./config.json" | awk 'BEGIN{RS="";getline<"-";print>ARGV[1]}' "./config.json"
71}
72
73# Shortcut to add additional uids and gids, based on the values set as part of
74# a rootless configuration.
75function runc_rootless_idmap() {
76 update_config ' .mounts |= map((select(.type == "devpts") | .options += ["gid=5"]) // .)
77 | .linux.uidMappings += [{"hostID": '"$ROOTLESS_UIDMAP_START"', "containerID": 1000, "size": '"$ROOTLESS_UIDMAP_LENGTH"'}]
78 | .linux.gidMappings += [{"hostID": '"$ROOTLESS_GIDMAP_START"', "containerID": 100, "size": 1}]
79 | .linux.gidMappings += [{"hostID": '"$((ROOTLESS_GIDMAP_START + 10))"', "containerID": 1, "size": 20}]
80 | .linux.gidMappings += [{"hostID": '"$((ROOTLESS_GIDMAP_START + 100))"', "containerID": 1000, "size": '"$((ROOTLESS_GIDMAP_LENGTH - 1000))"'}]'
81}
82
83# Returns systemd version as a number (-1 if systemd is not enabled/supported).
84function systemd_version() {
85 if [ -n "${RUNC_USE_SYSTEMD}" ]; then
86 systemctl --version | awk '/^systemd / {print $2; exit}'
87 return
88 fi
89
90 echo "-1"
91}
92
93function init_cgroup_paths() {
94 # init once
95 test -n "$CGROUP_UNIFIED" && return
96
97 if stat -f -c %t /sys/fs/cgroup | grep -qFw 63677270; then
98 CGROUP_UNIFIED=yes
99 local controllers="/sys/fs/cgroup/cgroup.controllers"
100 # For rootless + systemd case, controllers delegation is required,
101 # so check the controllers that the current user has, not the top one.
102 # NOTE: delegation of cpuset requires systemd >= 244 (Fedora >= 32, Ubuntu >= 20.04).
103 if [[ "$ROOTLESS" -ne 0 && -n "$RUNC_USE_SYSTEMD" ]]; then
104 controllers="/sys/fs/cgroup/user.slice/user-$(id -u).slice/user@$(id -u).service/cgroup.controllers"
105 fi
106
107 # "pseudo" controllers do not appear in /sys/fs/cgroup/cgroup.controllers.
108 # - devices (since kernel 4.15) we must assume to be supported because
109 # it's quite hard to test.
110 # - freezer (since kernel 5.2) we can auto-detect by looking for the
111 # "cgroup.freeze" file a *non-root* cgroup.
112 CGROUP_SUBSYSTEMS=$(
113 cat "$controllers"
114 echo devices
115 )
116 CGROUP_BASE_PATH=/sys/fs/cgroup
117
118 # Find any cgroup.freeze files...
119 if [ -n "$(find "$CGROUP_BASE_PATH" -type f -name "cgroup.freeze" -print -quit)" ]; then
120 CGROUP_SUBSYSTEMS+=" freezer"
121 fi
122 else
123 if stat -f -c %t /sys/fs/cgroup/unified | grep -qFw 63677270; then
124 CGROUP_HYBRID=yes
125 fi
126 CGROUP_UNIFIED=no
127 CGROUP_SUBSYSTEMS=$(awk '!/^#/ {print $1}' /proc/cgroups)
128 local g base_path
129 for g in ${CGROUP_SUBSYSTEMS}; do
130 base_path=$(gawk '$(NF-2) == "cgroup" && $NF ~ /\<'"${g}"'\>/ { print $5; exit }' /proc/self/mountinfo)
131 test -z "$base_path" && continue
132 eval CGROUP_"${g^^}"_BASE_PATH="${base_path}"
133 done
134 fi
135}
136
137function create_parent() {
138 if [ -n "$RUNC_USE_SYSTEMD" ]; then
139 [ -z "$SD_PARENT_NAME" ] && return
140 "$SD_HELPER" --parent machine.slice start "$SD_PARENT_NAME"
141 else
142 [ -z "$REL_PARENT_PATH" ] && return
143 if [ "$CGROUP_UNIFIED" == "yes" ]; then
144 mkdir "/sys/fs/cgroup$REL_PARENT_PATH"
145 else
146 local subsys
147 for subsys in ${CGROUP_SUBSYSTEMS}; do
148 # Have to ignore EEXIST (-p) as some subsystems
149 # are mounted together (e.g. cpu,cpuacct), so
150 # the path is created more than once.
151 mkdir -p "/sys/fs/cgroup/$subsys$REL_PARENT_PATH"
152 done
153 fi
154 fi
155}
156
157function remove_parent() {
158 if [ -n "$RUNC_USE_SYSTEMD" ]; then
159 [ -z "$SD_PARENT_NAME" ] && return
160 "$SD_HELPER" --parent machine.slice stop "$SD_PARENT_NAME"
161 else
162 [ -z "$REL_PARENT_PATH" ] && return
163 if [ "$CGROUP_UNIFIED" == "yes" ]; then
164 rmdir "/sys/fs/cgroup/$REL_PARENT_PATH"
165 else
166 local subsys
167 for subsys in ${CGROUP_SUBSYSTEMS} systemd; do
168 rmdir "/sys/fs/cgroup/$subsys/$REL_PARENT_PATH"
169 done
170 fi
171 fi
172 unset SD_PARENT_NAME
173 unset REL_PARENT_PATH
174}
175
176function set_parent_systemd_properties() {
177 [ -z "$SD_PARENT_NAME" ] && return
178 local user
179 [ "$(id -u)" != "0" ] && user="--user"
180 systemctl set-property $user "$SD_PARENT_NAME" "$@"
181}
182
183# Randomize cgroup path(s), and update cgroupsPath in config.json.
184# This function sets a few cgroup-related variables.
185#
186# Optional parameter $1 is a pod/parent name. If set, a parent/pod cgroup is
187# created, and variables $REL_PARENT_PATH and $SD_PARENT_NAME can be used to
188# refer to it.
189function set_cgroups_path() {
190 init_cgroup_paths
191 local pod dash_pod slash_pod pod_slice
192 if [ "$#" -ne 0 ] && [ "$1" != "" ]; then
193 # Set up a parent/pod cgroup.
194 pod="$1"
195 dash_pod="-$pod"
196 slash_pod="/$pod"
197 SD_PARENT_NAME="machine-${pod}.slice"
198 pod_slice="/$SD_PARENT_NAME"
199 fi
200
201 local rnd="$RANDOM"
202 if [ -n "${RUNC_USE_SYSTEMD}" ]; then
203 SD_UNIT_NAME="runc-cgroups-integration-test-${rnd}.scope"
204 if [ "$(id -u)" = "0" ]; then
205 REL_PARENT_PATH="/machine.slice${pod_slice}"
206 OCI_CGROUPS_PATH="machine${dash_pod}.slice:runc-cgroups:integration-test-${rnd}"
207 else
208 REL_PARENT_PATH="/user.slice/user-$(id -u).slice/user@$(id -u).service/machine.slice${pod_slice}"
209 # OCI path doesn't contain "/user.slice/user-$(id -u).slice/user@$(id -u).service/" prefix
210 OCI_CGROUPS_PATH="machine${dash_pod}.slice:runc-cgroups:integration-test-${rnd}"
211 fi
212 REL_CGROUPS_PATH="$REL_PARENT_PATH/$SD_UNIT_NAME"
213 else
214 REL_PARENT_PATH="/runc-cgroups-integration-test${slash_pod}"
215 REL_CGROUPS_PATH="$REL_PARENT_PATH/test-cgroup-${rnd}"
216 OCI_CGROUPS_PATH=$REL_CGROUPS_PATH
217 fi
218
219 # Absolute path to container's cgroup v2.
220 if [ "$CGROUP_UNIFIED" == "yes" ]; then
221 CGROUP_PATH=${CGROUP_BASE_PATH}${REL_CGROUPS_PATH}
222 fi
223
224 [ -n "$pod" ] && create_parent
225
226 update_config '.linux.cgroupsPath |= "'"${OCI_CGROUPS_PATH}"'"'
227}
228
229# Get a path to cgroup directory, based on controller name.
230# Parameters:
231# $1: controller name (like "pids") or a file name (like "pids.max").
232function get_cgroup_path() {
233 if [ "$CGROUP_UNIFIED" = "yes" ]; then
234 echo "$CGROUP_PATH"
235 return
236 fi
237
238 local var cgroup
239 var=${1%%.*} # controller name (e.g. memory)
240 var=CGROUP_${var^^}_BASE_PATH # variable name (e.g. CGROUP_MEMORY_BASE_PATH)
241 eval cgroup=\$"${var}${REL_CGROUPS_PATH}"
242 echo "$cgroup"
243}
244
245# Get a value from a cgroup file.
246function get_cgroup_value() {
247 local cgroup
248 cgroup="$(get_cgroup_path "$1")"
249 cat "$cgroup/$1"
250}
251
252# Helper to check a if value in a cgroup file matches the expected one.
253function check_cgroup_value() {
254 local current
255 current="$(get_cgroup_value "$1")"
256 local expected=$2
257
258 echo "current $current !? $expected"
259 [ "$current" = "$expected" ]
260}
261
262# Helper to check a value in systemd.
263function check_systemd_value() {
264 [ -z "${RUNC_USE_SYSTEMD}" ] && return
265 local source="$1"
266 [ "$source" = "unsupported" ] && return
267 local expected="$2"
268 local expected2="$3"
269 local user=""
270 [ "$(id -u)" != "0" ] && user="--user"
271
272 current=$(systemctl show $user --property "$source" "$SD_UNIT_NAME" | awk -F= '{print $2}')
273 echo "systemd $source: current $current !? $expected $expected2"
274 [ "$current" = "$expected" ] || [[ -n "$expected2" && "$current" = "$expected2" ]]
275}
276
277function check_cpu_quota() {
278 local quota=$1
279 local period=$2
280 local sd_quota=$3
281
282 if [ "$CGROUP_UNIFIED" = "yes" ]; then
283 if [ "$quota" = "-1" ]; then
284 quota="max"
285 fi
286 check_cgroup_value "cpu.max" "$quota $period"
287 else
288 check_cgroup_value "cpu.cfs_quota_us" $quota
289 check_cgroup_value "cpu.cfs_period_us" "$period"
290 fi
291 # systemd values are the same for v1 and v2
292 check_systemd_value "CPUQuotaPerSecUSec" "$sd_quota"
293
294 # CPUQuotaPeriodUSec requires systemd >= v242
295 [ "$(systemd_version)" -lt 242 ] && return
296
297 local sd_period=$((period / 1000))ms
298 [ "$sd_period" = "1000ms" ] && sd_period="1s"
299 local sd_infinity=""
300 # 100ms is the default value, and if not set, shown as infinity
301 [ "$sd_period" = "100ms" ] && sd_infinity="infinity"
302 check_systemd_value "CPUQuotaPeriodUSec" $sd_period $sd_infinity
303}
304
305# Works for cgroup v1 and v2, accepts v1 shares as an argument.
306function check_cpu_shares() {
307 local shares=$1
308
309 if [ "$CGROUP_UNIFIED" = "yes" ]; then
310 local weight=$((1 + ((shares - 2) * 9999) / 262142))
311 check_cpu_weight "$weight"
312 else
313 check_cgroup_value "cpu.shares" "$shares"
314 check_systemd_value "CPUShares" "$shares"
315 fi
316}
317
318# Works only for cgroup v2, accept v2 weight.
319function check_cpu_weight() {
320 local weight=$1
321
322 check_cgroup_value "cpu.weight" "$weight"
323 check_systemd_value "CPUWeight" "$weight"
324}
325
326# Helper function to set a resources limit
327function set_resources_limit() {
328 update_config '.linux.resources.pids.limit |= 100'
329}
330
331# Helper function to make /sys/fs/cgroup writable
332function set_cgroup_mount_writable() {
333 update_config '.mounts |= map((select(.type == "cgroup") | .options -= ["ro"]) // .)'
334}
335
336# Fails the current test, providing the error given.
337function fail() {
338 echo "$@" >&2
339 exit 1
340}
341
342# Check whether rootless runc can use cgroups.
343function rootless_cgroup() {
344 [[ "$ROOTLESS_FEATURES" == *"cgroup"* || -n "$RUNC_USE_SYSTEMD" ]]
345}
346
347# Check if criu is available and working.
348function have_criu() {
349 command -v criu &>/dev/null || return 1
350
351 # Workaround for https://github.com/opencontainers/runc/issues/3532.
352 local ver
353 ver=$(rpm -q criu 2>/dev/null || true)
354 run ! grep -q '^criu-3\.17-[123]\.el9' <<<"$ver"
355}
356
357# Allows a test to specify what things it requires. If the environment can't
358# support it, the test is skipped with a message.
359function requires() {
360 for var in "$@"; do
361 local skip_me
362 case $var in
363 criu)
364 if ! have_criu; then
365 skip_me=1
366 fi
367 ;;
368 root)
369 if [ "$ROOTLESS" -ne 0 ]; then
370 skip_me=1
371 fi
372 ;;
373 rootless)
374 if [ "$ROOTLESS" -eq 0 ]; then
375 skip_me=1
376 fi
377 ;;
378 rootless_idmap)
379 if [[ "$ROOTLESS_FEATURES" != *"idmap"* ]]; then
380 skip_me=1
381 fi
382 ;;
383 rootless_cgroup)
384 if ! rootless_cgroup; then
385 skip_me=1
386 fi
387 ;;
388 rootless_no_cgroup)
389 if rootless_cgroup; then
390 skip_me=1
391 fi
392 ;;
393 rootless_no_features)
394 if [ "$ROOTLESS_FEATURES" != "" ]; then
395 skip_me=1
396 fi
397 ;;
398 cgroups_rt)
399 init_cgroup_paths
400 if [ ! -e "${CGROUP_CPU_BASE_PATH}/cpu.rt_period_us" ]; then
401 skip_me=1
402 fi
403 ;;
404 cgroups_swap)
405 init_cgroup_paths
406 if [ $CGROUP_UNIFIED = "no" ] && [ ! -e "${CGROUP_MEMORY_BASE_PATH}/memory.memsw.limit_in_bytes" ]; then
407 skip_me=1
408 fi
409 ;;
410 cgroupns)
411 if [ ! -e "/proc/self/ns/cgroup" ]; then
412 skip_me=1
413 fi
414 ;;
415 cgroups_v1)
416 init_cgroup_paths
417 if [ "$CGROUP_UNIFIED" != "no" ]; then
418 skip_me=1
419 fi
420 ;;
421 cgroups_v2)
422 init_cgroup_paths
423 if [ "$CGROUP_UNIFIED" != "yes" ]; then
424 skip_me=1
425 fi
426 ;;
427 cgroups_hybrid)
428 init_cgroup_paths
429 if [ "$CGROUP_HYBRID" != "yes" ]; then
430 skip_me=1
431 fi
432 ;;
433 cgroups_*)
434 init_cgroup_paths
435 var=${var#cgroups_}
436 if [[ "$CGROUP_SUBSYSTEMS" != *"$var"* ]]; then
437 skip_me=1
438 fi
439 ;;
440 smp)
441 local cpus
442 cpus=$(grep -c '^processor' /proc/cpuinfo)
443 if [ "$cpus" -lt 2 ]; then
444 skip_me=1
445 fi
446 ;;
447 systemd)
448 if [ -z "${RUNC_USE_SYSTEMD}" ]; then
449 skip_me=1
450 fi
451 ;;
452 systemd_v*)
453 var=${var#systemd_v}
454 if [ "$(systemd_version)" -lt "$var" ]; then
455 skip "requires systemd >= v${var}"
456 fi
457 ;;
458 no_systemd)
459 if [ -n "${RUNC_USE_SYSTEMD}" ]; then
460 skip_me=1
461 fi
462 ;;
463 arch_x86_64)
464 if [ "$ARCH" != "x86_64" ]; then
465 skip_me=1
466 fi
467 ;;
468 more_than_8_core)
469 local cpus
470 cpus=$(grep -c '^processor' /proc/cpuinfo)
471 if [ "$cpus" -le 8 ]; then
472 skip_me=1
473 fi
474 ;;
475 *)
476 fail "BUG: Invalid requires $var."
477 ;;
478 esac
479 if [ -n "$skip_me" ]; then
480 skip "test requires $var"
481 fi
482 done
483}
484
485# Retry a command $1 times until it succeeds. Wait $2 seconds between retries.
486function retry() {
487 local attempts=$1
488 shift
489 local delay=$1
490 shift
491 local i
492
493 for ((i = 0; i < attempts; i++)); do
494 run "$@"
495 if [[ "$status" -eq 0 ]]; then
496 return 0
497 fi
498 sleep "$delay"
499 done
500
501 echo "Command \"$*\" failed $attempts times. Output: $output"
502 false
503}
504
505# retry until the given container has state
506function wait_for_container() {
507 if [ $# -eq 3 ]; then
508 retry "$1" "$2" __runc state "$3"
509 elif [ $# -eq 4 ]; then
510 retry "$1" "$2" eval "__runc state $3 | grep -qw $4"
511 else
512 echo "Usage: wait_for_container ATTEMPTS DELAY ID [STATUS]" 1>&2
513 return 1
514 fi
515}
516
517function testcontainer() {
518 # test state of container
519 runc state "$1"
520 if [ "$2" == "checkpointed" ]; then
521 [ "$status" -eq 1 ]
522 return
523 fi
524 [ "$status" -eq 0 ]
525 [[ "${output}" == *"$2"* ]]
526}
527
528function setup_recvtty() {
529 [ -z "$ROOT" ] && return 1 # must not be called without ROOT set
530 local dir="$ROOT/tty"
531
532 mkdir "$dir"
533 export CONSOLE_SOCKET="$dir/sock"
534
535 # We need to start recvtty in the background, so we double fork in the shell.
536 ("$RECVTTY" --pid-file "$dir/pid" --mode null "$CONSOLE_SOCKET" &) &
537}
538
539function teardown_recvtty() {
540 [ -z "$ROOT" ] && return 0 # nothing to teardown
541 local dir="$ROOT/tty"
542
543 # When we kill recvtty, the container will also be killed.
544 if [ -f "$dir/pid" ]; then
545 kill -9 "$(cat "$dir/pid")"
546 fi
547
548 # Clean up the files that might be left over.
549 rm -rf "$dir"
550}
551
552function setup_seccompagent() {
553 ("${SECCOMP_AGENT}" -socketfile="$SECCCOMP_AGENT_SOCKET" -pid-file "$BATS_TMPDIR/seccompagent.pid" &) &
554}
555
556function teardown_seccompagent() {
557 if [ -f "$BATS_TMPDIR/seccompagent.pid" ]; then
558 kill -9 "$(cat "$BATS_TMPDIR/seccompagent.pid")"
559 fi
560 rm -f "$BATS_TMPDIR/seccompagent.pid"
561 rm -f "$SECCCOMP_AGENT_SOCKET"
562}
563
564function setup_bundle() {
565 local image="$1"
566
567 # Root for various container directories (state, tty, bundle).
568 ROOT=$(mktemp -d "$BATS_RUN_TMPDIR/runc.XXXXXX")
569 mkdir -p "$ROOT/state" "$ROOT/bundle/rootfs"
570
571 # Directories created by mktemp -d have 0700 permission bits. Tests
572 # running inside userns (see userns.bats) need to access the directory
573 # as a different user to mount the rootfs. Since kernel v5.12, parent
574 # directories are also checked. Give a+x for these tests to work.
575 chmod a+x "$ROOT" "$BATS_RUN_TMPDIR"
576
577 setup_recvtty
578 cd "$ROOT/bundle" || return
579
580 tar --exclude './dev/*' -C rootfs -xf "$image"
581
582 runc_spec
583}
584
585function setup_busybox() {
586 setup_bundle "$BUSYBOX_IMAGE"
587}
588
589function setup_debian() {
590 setup_bundle "$DEBIAN_IMAGE"
591}
592
593function teardown_bundle() {
594 [ -z "$ROOT" ] && return 0 # nothing to teardown
595
596 cd "$INTEGRATION_ROOT" || return
597 teardown_recvtty
598 local ct
599 for ct in $(__runc list -q); do
600 __runc delete -f "$ct"
601 done
602 rm -rf "$ROOT"
603 remove_parent
604}
605
606function requires_kernel() {
607 local major_required minor_required
608 major_required=$(echo "$1" | cut -d. -f1)
609 minor_required=$(echo "$1" | cut -d. -f2)
610 if [[ "$KERNEL_MAJOR" -lt $major_required || ("$KERNEL_MAJOR" -eq $major_required && "$KERNEL_MINOR" -lt $minor_required) ]]; then
611 skip "requires kernel $1"
612 fi
613}
View as plain text