blkio.go

Documentation: github.com/containerd/cgroups

     1  /*
     2     Copyright The containerd Authors.
     3  
     4     Licensed under the Apache License, Version 2.0 (the "License");
     5     you may not use this file except in compliance with the License.
     6     You may obtain a copy of the License at
     7  
     8         http://www.apache.org/licenses/LICENSE-2.0
     9  
    10     Unless required by applicable law or agreed to in writing, software
    11     distributed under the License is distributed on an "AS IS" BASIS,
    12     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13     See the License for the specific language governing permissions and
    14     limitations under the License.
    15  */
    16  
    17  package cgroups
    18  
    19  import (
    20  	"bufio"
    21  	"fmt"
    22  	"io"
    23  	"os"
    24  	"path/filepath"
    25  	"strconv"
    26  	"strings"
    27  
    28  	v1 "github.com/containerd/cgroups/stats/v1"
    29  	specs "github.com/opencontainers/runtime-spec/specs-go"
    30  )
    31  
    32  // NewBlkio returns a Blkio controller given the root folder of cgroups.
    33  // It may optionally accept other configuration options, such as ProcRoot(path)
    34  func NewBlkio(root string, options ...func(controller *blkioController)) *blkioController {
    35  	ctrl := &blkioController{
    36  		root:     filepath.Join(root, string(Blkio)),
    37  		procRoot: "/proc",
    38  	}
    39  	for _, opt := range options {
    40  		opt(ctrl)
    41  	}
    42  	return ctrl
    43  }
    44  
    45  // ProcRoot overrides the default location of the "/proc" filesystem
    46  func ProcRoot(path string) func(controller *blkioController) {
    47  	return func(c *blkioController) {
    48  		c.procRoot = path
    49  	}
    50  }
    51  
    52  type blkioController struct {
    53  	root     string
    54  	procRoot string
    55  }
    56  
    57  func (b *blkioController) Name() Name {
    58  	return Blkio
    59  }
    60  
    61  func (b *blkioController) Path(path string) string {
    62  	return filepath.Join(b.root, path)
    63  }
    64  
    65  func (b *blkioController) Create(path string, resources *specs.LinuxResources) error {
    66  	if err := os.MkdirAll(b.Path(path), defaultDirPerm); err != nil {
    67  		return err
    68  	}
    69  	if resources.BlockIO == nil {
    70  		return nil
    71  	}
    72  	for _, t := range createBlkioSettings(resources.BlockIO) {
    73  		if t.value != nil {
    74  			if err := retryingWriteFile(
    75  				filepath.Join(b.Path(path), "blkio."+t.name),
    76  				t.format(t.value),
    77  				defaultFilePerm,
    78  			); err != nil {
    79  				return err
    80  			}
    81  		}
    82  	}
    83  	return nil
    84  }
    85  
    86  func (b *blkioController) Update(path string, resources *specs.LinuxResources) error {
    87  	return b.Create(path, resources)
    88  }
    89  
    90  func (b *blkioController) Stat(path string, stats *v1.Metrics) error {
    91  	stats.Blkio = &v1.BlkIOStat{}
    92  
    93  	var settings []blkioStatSettings
    94  
    95  	// Try to read CFQ stats available on all CFQ enabled kernels first
    96  	if _, err := os.Lstat(filepath.Join(b.Path(path), "blkio.io_serviced_recursive")); err == nil {
    97  		settings = []blkioStatSettings{
    98  			{
    99  				name:  "sectors_recursive",
   100  				entry: &stats.Blkio.SectorsRecursive,
   101  			},
   102  			{
   103  				name:  "io_service_bytes_recursive",
   104  				entry: &stats.Blkio.IoServiceBytesRecursive,
   105  			},
   106  			{
   107  				name:  "io_serviced_recursive",
   108  				entry: &stats.Blkio.IoServicedRecursive,
   109  			},
   110  			{
   111  				name:  "io_queued_recursive",
   112  				entry: &stats.Blkio.IoQueuedRecursive,
   113  			},
   114  			{
   115  				name:  "io_service_time_recursive",
   116  				entry: &stats.Blkio.IoServiceTimeRecursive,
   117  			},
   118  			{
   119  				name:  "io_wait_time_recursive",
   120  				entry: &stats.Blkio.IoWaitTimeRecursive,
   121  			},
   122  			{
   123  				name:  "io_merged_recursive",
   124  				entry: &stats.Blkio.IoMergedRecursive,
   125  			},
   126  			{
   127  				name:  "time_recursive",
   128  				entry: &stats.Blkio.IoTimeRecursive,
   129  			},
   130  		}
   131  	}
   132  
   133  	f, err := os.Open(filepath.Join(b.procRoot, "partitions"))
   134  	if err != nil {
   135  		return err
   136  	}
   137  	defer f.Close()
   138  
   139  	devices, err := getDevices(f)
   140  	if err != nil {
   141  		return err
   142  	}
   143  
   144  	var size int
   145  	for _, t := range settings {
   146  		if err := b.readEntry(devices, path, t.name, t.entry); err != nil {
   147  			return err
   148  		}
   149  		size += len(*t.entry)
   150  	}
   151  	if size > 0 {
   152  		return nil
   153  	}
   154  
   155  	// Even the kernel is compiled with the CFQ scheduler, the cgroup may not use
   156  	// block devices with the CFQ scheduler. If so, we should fallback to throttle.* files.
   157  	settings = []blkioStatSettings{
   158  		{
   159  			name:  "throttle.io_serviced",
   160  			entry: &stats.Blkio.IoServicedRecursive,
   161  		},
   162  		{
   163  			name:  "throttle.io_service_bytes",
   164  			entry: &stats.Blkio.IoServiceBytesRecursive,
   165  		},
   166  	}
   167  	for _, t := range settings {
   168  		if err := b.readEntry(devices, path, t.name, t.entry); err != nil {
   169  			return err
   170  		}
   171  	}
   172  	return nil
   173  }
   174  
   175  func (b *blkioController) readEntry(devices map[deviceKey]string, path, name string, entry *[]*v1.BlkIOEntry) error {
   176  	f, err := os.Open(filepath.Join(b.Path(path), "blkio."+name))
   177  	if err != nil {
   178  		return err
   179  	}
   180  	defer f.Close()
   181  	sc := bufio.NewScanner(f)
   182  	for sc.Scan() {
   183  		// format: dev type amount
   184  		fields := strings.FieldsFunc(sc.Text(), splitBlkIOStatLine)
   185  		if len(fields) < 3 {
   186  			if len(fields) == 2 && fields[0] == "Total" {
   187  				// skip total line
   188  				continue
   189  			} else {
   190  				return fmt.Errorf("invalid line found while parsing %s: %s", path, sc.Text())
   191  			}
   192  		}
   193  		major, err := strconv.ParseUint(fields[0], 10, 64)
   194  		if err != nil {
   195  			return err
   196  		}
   197  		minor, err := strconv.ParseUint(fields[1], 10, 64)
   198  		if err != nil {
   199  			return err
   200  		}
   201  		op := ""
   202  		valueField := 2
   203  		if len(fields) == 4 {
   204  			op = fields[2]
   205  			valueField = 3
   206  		}
   207  		v, err := strconv.ParseUint(fields[valueField], 10, 64)
   208  		if err != nil {
   209  			return err
   210  		}
   211  		*entry = append(*entry, &v1.BlkIOEntry{
   212  			Device: devices[deviceKey{major, minor}],
   213  			Major:  major,
   214  			Minor:  minor,
   215  			Op:     op,
   216  			Value:  v,
   217  		})
   218  	}
   219  	return sc.Err()
   220  }
   221  
   222  func createBlkioSettings(blkio *specs.LinuxBlockIO) []blkioSettings {
   223  	settings := []blkioSettings{}
   224  
   225  	if blkio.Weight != nil {
   226  		settings = append(settings,
   227  			blkioSettings{
   228  				name:   "weight",
   229  				value:  blkio.Weight,
   230  				format: uintf,
   231  			})
   232  	}
   233  	if blkio.LeafWeight != nil {
   234  		settings = append(settings,
   235  			blkioSettings{
   236  				name:   "leaf_weight",
   237  				value:  blkio.LeafWeight,
   238  				format: uintf,
   239  			})
   240  	}
   241  	for _, wd := range blkio.WeightDevice {
   242  		if wd.Weight != nil {
   243  			settings = append(settings,
   244  				blkioSettings{
   245  					name:   "weight_device",
   246  					value:  wd,
   247  					format: weightdev,
   248  				})
   249  		}
   250  		if wd.LeafWeight != nil {
   251  			settings = append(settings,
   252  				blkioSettings{
   253  					name:   "leaf_weight_device",
   254  					value:  wd,
   255  					format: weightleafdev,
   256  				})
   257  		}
   258  	}
   259  	for _, t := range []struct {
   260  		name string
   261  		list []specs.LinuxThrottleDevice
   262  	}{
   263  		{
   264  			name: "throttle.read_bps_device",
   265  			list: blkio.ThrottleReadBpsDevice,
   266  		},
   267  		{
   268  			name: "throttle.read_iops_device",
   269  			list: blkio.ThrottleReadIOPSDevice,
   270  		},
   271  		{
   272  			name: "throttle.write_bps_device",
   273  			list: blkio.ThrottleWriteBpsDevice,
   274  		},
   275  		{
   276  			name: "throttle.write_iops_device",
   277  			list: blkio.ThrottleWriteIOPSDevice,
   278  		},
   279  	} {
   280  		for _, td := range t.list {
   281  			settings = append(settings, blkioSettings{
   282  				name:   t.name,
   283  				value:  td,
   284  				format: throttleddev,
   285  			})
   286  		}
   287  	}
   288  	return settings
   289  }
   290  
   291  type blkioSettings struct {
   292  	name   string
   293  	value  interface{}
   294  	format func(v interface{}) []byte
   295  }
   296  
   297  type blkioStatSettings struct {
   298  	name  string
   299  	entry *[]*v1.BlkIOEntry
   300  }
   301  
   302  func uintf(v interface{}) []byte {
   303  	return []byte(strconv.FormatUint(uint64(*v.(*uint16)), 10))
   304  }
   305  
   306  func weightdev(v interface{}) []byte {
   307  	wd := v.(specs.LinuxWeightDevice)
   308  	return []byte(fmt.Sprintf("%d:%d %d", wd.Major, wd.Minor, *wd.Weight))
   309  }
   310  
   311  func weightleafdev(v interface{}) []byte {
   312  	wd := v.(specs.LinuxWeightDevice)
   313  	return []byte(fmt.Sprintf("%d:%d %d", wd.Major, wd.Minor, *wd.LeafWeight))
   314  }
   315  
   316  func throttleddev(v interface{}) []byte {
   317  	td := v.(specs.LinuxThrottleDevice)
   318  	return []byte(fmt.Sprintf("%d:%d %d", td.Major, td.Minor, td.Rate))
   319  }
   320  
   321  func splitBlkIOStatLine(r rune) bool {
   322  	return r == ' ' || r == ':'
   323  }
   324  
   325  type deviceKey struct {
   326  	major, minor uint64
   327  }
   328  
   329  // getDevices makes a best effort attempt to read all the devices into a map
   330  // keyed by major and minor number. Since devices may be mapped multiple times,
   331  // we err on taking the first occurrence.
   332  func getDevices(r io.Reader) (map[deviceKey]string, error) {
   333  
   334  	var (
   335  		s       = bufio.NewScanner(r)
   336  		devices = make(map[deviceKey]string)
   337  	)
   338  	for i := 0; s.Scan(); i++ {
   339  		if i < 2 {
   340  			continue
   341  		}
   342  		fields := strings.Fields(s.Text())
   343  		major, err := strconv.Atoi(fields[0])
   344  		if err != nil {
   345  			return nil, err
   346  		}
   347  		minor, err := strconv.Atoi(fields[1])
   348  		if err != nil {
   349  			return nil, err
   350  		}
   351  		key := deviceKey{
   352  			major: uint64(major),
   353  			minor: uint64(minor),
   354  		}
   355  		if _, ok := devices[key]; ok {
   356  			continue
   357  		}
   358  		devices[key] = filepath.Join("/dev", fields[3])
   359  	}
   360  	return devices, s.Err()
   361  }
   362
View as plain text