memory.go

Documentation: github.com/opencontainers/runc/libcontainer/cgroups/fs

     1  package fs
     2  
     3  import (
     4  	"bufio"
     5  	"errors"
     6  	"fmt"
     7  	"math"
     8  	"os"
     9  	"path/filepath"
    10  	"strconv"
    11  	"strings"
    12  
    13  	"golang.org/x/sys/unix"
    14  
    15  	"github.com/opencontainers/runc/libcontainer/cgroups"
    16  	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
    17  	"github.com/opencontainers/runc/libcontainer/configs"
    18  )
    19  
    20  const (
    21  	cgroupMemorySwapLimit = "memory.memsw.limit_in_bytes"
    22  	cgroupMemoryLimit     = "memory.limit_in_bytes"
    23  	cgroupMemoryUsage     = "memory.usage_in_bytes"
    24  	cgroupMemoryMaxUsage  = "memory.max_usage_in_bytes"
    25  )
    26  
    27  type MemoryGroup struct{}
    28  
    29  func (s *MemoryGroup) Name() string {
    30  	return "memory"
    31  }
    32  
    33  func (s *MemoryGroup) Apply(path string, _ *configs.Resources, pid int) error {
    34  	return apply(path, pid)
    35  }
    36  
    37  func setMemory(path string, val int64) error {
    38  	if val == 0 {
    39  		return nil
    40  	}
    41  
    42  	err := cgroups.WriteFile(path, cgroupMemoryLimit, strconv.FormatInt(val, 10))
    43  	if !errors.Is(err, unix.EBUSY) {
    44  		return err
    45  	}
    46  
    47  	// EBUSY means the kernel can't set new limit as it's too low
    48  	// (lower than the current usage). Return more specific error.
    49  	usage, err := fscommon.GetCgroupParamUint(path, cgroupMemoryUsage)
    50  	if err != nil {
    51  		return err
    52  	}
    53  	max, err := fscommon.GetCgroupParamUint(path, cgroupMemoryMaxUsage)
    54  	if err != nil {
    55  		return err
    56  	}
    57  
    58  	return fmt.Errorf("unable to set memory limit to %d (current usage: %d, peak usage: %d)", val, usage, max)
    59  }
    60  
    61  func setSwap(path string, val int64) error {
    62  	if val == 0 {
    63  		return nil
    64  	}
    65  
    66  	return cgroups.WriteFile(path, cgroupMemorySwapLimit, strconv.FormatInt(val, 10))
    67  }
    68  
    69  func setMemoryAndSwap(path string, r *configs.Resources) error {
    70  	// If the memory update is set to -1 and the swap is not explicitly
    71  	// set, we should also set swap to -1, it means unlimited memory.
    72  	if r.Memory == -1 && r.MemorySwap == 0 {
    73  		// Only set swap if it's enabled in kernel
    74  		if cgroups.PathExists(filepath.Join(path, cgroupMemorySwapLimit)) {
    75  			r.MemorySwap = -1
    76  		}
    77  	}
    78  
    79  	// When memory and swap memory are both set, we need to handle the cases
    80  	// for updating container.
    81  	if r.Memory != 0 && r.MemorySwap != 0 {
    82  		curLimit, err := fscommon.GetCgroupParamUint(path, cgroupMemoryLimit)
    83  		if err != nil {
    84  			return err
    85  		}
    86  
    87  		// When update memory limit, we should adapt the write sequence
    88  		// for memory and swap memory, so it won't fail because the new
    89  		// value and the old value don't fit kernel's validation.
    90  		if r.MemorySwap == -1 || curLimit < uint64(r.MemorySwap) {
    91  			if err := setSwap(path, r.MemorySwap); err != nil {
    92  				return err
    93  			}
    94  			if err := setMemory(path, r.Memory); err != nil {
    95  				return err
    96  			}
    97  			return nil
    98  		}
    99  	}
   100  
   101  	if err := setMemory(path, r.Memory); err != nil {
   102  		return err
   103  	}
   104  	if err := setSwap(path, r.MemorySwap); err != nil {
   105  		return err
   106  	}
   107  
   108  	return nil
   109  }
   110  
   111  func (s *MemoryGroup) Set(path string, r *configs.Resources) error {
   112  	if err := setMemoryAndSwap(path, r); err != nil {
   113  		return err
   114  	}
   115  
   116  	// ignore KernelMemory and KernelMemoryTCP
   117  
   118  	if r.MemoryReservation != 0 {
   119  		if err := cgroups.WriteFile(path, "memory.soft_limit_in_bytes", strconv.FormatInt(r.MemoryReservation, 10)); err != nil {
   120  			return err
   121  		}
   122  	}
   123  
   124  	if r.OomKillDisable {
   125  		if err := cgroups.WriteFile(path, "memory.oom_control", "1"); err != nil {
   126  			return err
   127  		}
   128  	}
   129  	if r.MemorySwappiness == nil || int64(*r.MemorySwappiness) == -1 {
   130  		return nil
   131  	} else if *r.MemorySwappiness <= 100 {
   132  		if err := cgroups.WriteFile(path, "memory.swappiness", strconv.FormatUint(*r.MemorySwappiness, 10)); err != nil {
   133  			return err
   134  		}
   135  	} else {
   136  		return fmt.Errorf("invalid memory swappiness value: %d (valid range is 0-100)", *r.MemorySwappiness)
   137  	}
   138  
   139  	return nil
   140  }
   141  
   142  func (s *MemoryGroup) GetStats(path string, stats *cgroups.Stats) error {
   143  	const file = "memory.stat"
   144  	statsFile, err := cgroups.OpenFile(path, file, os.O_RDONLY)
   145  	if err != nil {
   146  		if os.IsNotExist(err) {
   147  			return nil
   148  		}
   149  		return err
   150  	}
   151  	defer statsFile.Close()
   152  
   153  	sc := bufio.NewScanner(statsFile)
   154  	for sc.Scan() {
   155  		t, v, err := fscommon.ParseKeyValue(sc.Text())
   156  		if err != nil {
   157  			return &parseError{Path: path, File: file, Err: err}
   158  		}
   159  		stats.MemoryStats.Stats[t] = v
   160  	}
   161  	stats.MemoryStats.Cache = stats.MemoryStats.Stats["cache"]
   162  
   163  	memoryUsage, err := getMemoryData(path, "")
   164  	if err != nil {
   165  		return err
   166  	}
   167  	stats.MemoryStats.Usage = memoryUsage
   168  	swapUsage, err := getMemoryData(path, "memsw")
   169  	if err != nil {
   170  		return err
   171  	}
   172  	stats.MemoryStats.SwapUsage = swapUsage
   173  	stats.MemoryStats.SwapOnlyUsage = cgroups.MemoryData{
   174  		Usage:   swapUsage.Usage - memoryUsage.Usage,
   175  		Failcnt: swapUsage.Failcnt - memoryUsage.Failcnt,
   176  	}
   177  	kernelUsage, err := getMemoryData(path, "kmem")
   178  	if err != nil {
   179  		return err
   180  	}
   181  	stats.MemoryStats.KernelUsage = kernelUsage
   182  	kernelTCPUsage, err := getMemoryData(path, "kmem.tcp")
   183  	if err != nil {
   184  		return err
   185  	}
   186  	stats.MemoryStats.KernelTCPUsage = kernelTCPUsage
   187  
   188  	value, err := fscommon.GetCgroupParamUint(path, "memory.use_hierarchy")
   189  	if err != nil {
   190  		return err
   191  	}
   192  	if value == 1 {
   193  		stats.MemoryStats.UseHierarchy = true
   194  	}
   195  
   196  	pagesByNUMA, err := getPageUsageByNUMA(path)
   197  	if err != nil {
   198  		return err
   199  	}
   200  	stats.MemoryStats.PageUsageByNUMA = pagesByNUMA
   201  
   202  	return nil
   203  }
   204  
   205  func getMemoryData(path, name string) (cgroups.MemoryData, error) {
   206  	memoryData := cgroups.MemoryData{}
   207  
   208  	moduleName := "memory"
   209  	if name != "" {
   210  		moduleName = "memory." + name
   211  	}
   212  	var (
   213  		usage    = moduleName + ".usage_in_bytes"
   214  		maxUsage = moduleName + ".max_usage_in_bytes"
   215  		failcnt  = moduleName + ".failcnt"
   216  		limit    = moduleName + ".limit_in_bytes"
   217  	)
   218  
   219  	value, err := fscommon.GetCgroupParamUint(path, usage)
   220  	if err != nil {
   221  		if name != "" && os.IsNotExist(err) {
   222  			// Ignore ENOENT as swap and kmem controllers
   223  			// are optional in the kernel.
   224  			return cgroups.MemoryData{}, nil
   225  		}
   226  		return cgroups.MemoryData{}, err
   227  	}
   228  	memoryData.Usage = value
   229  	value, err = fscommon.GetCgroupParamUint(path, maxUsage)
   230  	if err != nil {
   231  		return cgroups.MemoryData{}, err
   232  	}
   233  	memoryData.MaxUsage = value
   234  	value, err = fscommon.GetCgroupParamUint(path, failcnt)
   235  	if err != nil {
   236  		return cgroups.MemoryData{}, err
   237  	}
   238  	memoryData.Failcnt = value
   239  	value, err = fscommon.GetCgroupParamUint(path, limit)
   240  	if err != nil {
   241  		if name == "kmem" && os.IsNotExist(err) {
   242  			// Ignore ENOENT as kmem.limit_in_bytes has
   243  			// been removed in newer kernels.
   244  			return memoryData, nil
   245  		}
   246  
   247  		return cgroups.MemoryData{}, err
   248  	}
   249  	memoryData.Limit = value
   250  
   251  	return memoryData, nil
   252  }
   253  
   254  func getPageUsageByNUMA(path string) (cgroups.PageUsageByNUMA, error) {
   255  	const (
   256  		maxColumns = math.MaxUint8 + 1
   257  		file       = "memory.numa_stat"
   258  	)
   259  	stats := cgroups.PageUsageByNUMA{}
   260  
   261  	fd, err := cgroups.OpenFile(path, file, os.O_RDONLY)
   262  	if os.IsNotExist(err) {
   263  		return stats, nil
   264  	} else if err != nil {
   265  		return stats, err
   266  	}
   267  	defer fd.Close()
   268  
   269  	// File format is documented in linux/Documentation/cgroup-v1/memory.txt
   270  	// and it looks like this:
   271  	//
   272  	// total=<total pages> N0=<node 0 pages> N1=<node 1 pages> ...
   273  	// file=<total file pages> N0=<node 0 pages> N1=<node 1 pages> ...
   274  	// anon=<total anon pages> N0=<node 0 pages> N1=<node 1 pages> ...
   275  	// unevictable=<total anon pages> N0=<node 0 pages> N1=<node 1 pages> ...
   276  	// hierarchical_<counter>=<counter pages> N0=<node 0 pages> N1=<node 1 pages> ...
   277  
   278  	scanner := bufio.NewScanner(fd)
   279  	for scanner.Scan() {
   280  		var field *cgroups.PageStats
   281  
   282  		line := scanner.Text()
   283  		columns := strings.SplitN(line, " ", maxColumns)
   284  		for i, column := range columns {
   285  			byNode := strings.SplitN(column, "=", 2)
   286  			// Some custom kernels have non-standard fields, like
   287  			//   numa_locality 0 0 0 0 0 0 0 0 0 0
   288  			//   numa_exectime 0
   289  			if len(byNode) < 2 {
   290  				if i == 0 {
   291  					// Ignore/skip those.
   292  					break
   293  				} else {
   294  					// The first column was already validated,
   295  					// so be strict to the rest.
   296  					return stats, malformedLine(path, file, line)
   297  				}
   298  			}
   299  			key, val := byNode[0], byNode[1]
   300  			if i == 0 { // First column: key is name, val is total.
   301  				field = getNUMAField(&stats, key)
   302  				if field == nil { // unknown field (new kernel?)
   303  					break
   304  				}
   305  				field.Total, err = strconv.ParseUint(val, 0, 64)
   306  				if err != nil {
   307  					return stats, &parseError{Path: path, File: file, Err: err}
   308  				}
   309  				field.Nodes = map[uint8]uint64{}
   310  			} else { // Subsequent columns: key is N<id>, val is usage.
   311  				if len(key) < 2 || key[0] != 'N' {
   312  					// This is definitely an error.
   313  					return stats, malformedLine(path, file, line)
   314  				}
   315  
   316  				n, err := strconv.ParseUint(key[1:], 10, 8)
   317  				if err != nil {
   318  					return stats, &parseError{Path: path, File: file, Err: err}
   319  				}
   320  
   321  				usage, err := strconv.ParseUint(val, 10, 64)
   322  				if err != nil {
   323  					return stats, &parseError{Path: path, File: file, Err: err}
   324  				}
   325  
   326  				field.Nodes[uint8(n)] = usage
   327  			}
   328  
   329  		}
   330  	}
   331  	if err := scanner.Err(); err != nil {
   332  		return cgroups.PageUsageByNUMA{}, &parseError{Path: path, File: file, Err: err}
   333  	}
   334  
   335  	return stats, nil
   336  }
   337  
   338  func getNUMAField(stats *cgroups.PageUsageByNUMA, name string) *cgroups.PageStats {
   339  	switch name {
   340  	case "total":
   341  		return &stats.Total
   342  	case "file":
   343  		return &stats.File
   344  	case "anon":
   345  		return &stats.Anon
   346  	case "unevictable":
   347  		return &stats.Unevictable
   348  	case "hierarchical_total":
   349  		return &stats.Hierarchical.Total
   350  	case "hierarchical_file":
   351  		return &stats.Hierarchical.File
   352  	case "hierarchical_anon":
   353  		return &stats.Hierarchical.Anon
   354  	case "hierarchical_unevictable":
   355  		return &stats.Hierarchical.Unevictable
   356  	}
   357  	return nil
   358  }
   359
View as plain text