quota_linux.go

Documentation: k8s.io/kubernetes/pkg/volume/util/fsquota

     1  //go:build linux
     2  // +build linux
     3  
     4  /*
     5  Copyright 2018 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package fsquota
    21  
    22  import (
    23  	"bufio"
    24  	"fmt"
    25  	"os"
    26  	"path/filepath"
    27  	"sync"
    28  
    29  	"k8s.io/klog/v2"
    30  	"k8s.io/mount-utils"
    31  
    32  	"k8s.io/apimachinery/pkg/api/resource"
    33  	"k8s.io/apimachinery/pkg/types"
    34  	"k8s.io/apimachinery/pkg/util/uuid"
    35  	"k8s.io/kubernetes/pkg/volume/util/fsquota/common"
    36  )
    37  
    38  // Pod -> External Pod UID
    39  var podUidMap = make(map[types.UID]types.UID)
    40  
    41  // Pod -> ID
    42  var podQuotaMap = make(map[types.UID]common.QuotaID)
    43  
    44  // Dir -> ID (for convenience)
    45  var dirQuotaMap = make(map[string]common.QuotaID)
    46  
    47  // ID -> pod
    48  var quotaPodMap = make(map[common.QuotaID]types.UID)
    49  
    50  // Directory -> pod
    51  var dirPodMap = make(map[string]types.UID)
    52  
    53  // Backing device -> applier
    54  // This is *not* cleaned up; its size will be bounded.
    55  var devApplierMap = make(map[string]common.LinuxVolumeQuotaApplier)
    56  
    57  // Directory -> applier
    58  var dirApplierMap = make(map[string]common.LinuxVolumeQuotaApplier)
    59  var dirApplierLock sync.RWMutex
    60  
    61  // Pod -> refcount
    62  var podDirCountMap = make(map[types.UID]int)
    63  
    64  // ID -> size
    65  var quotaSizeMap = make(map[common.QuotaID]int64)
    66  var quotaLock sync.RWMutex
    67  
    68  var supportsQuotasMap = make(map[string]bool)
    69  var supportsQuotasLock sync.RWMutex
    70  
    71  // Directory -> backingDev
    72  var backingDevMap = make(map[string]string)
    73  var backingDevLock sync.RWMutex
    74  
    75  var mountpointMap = make(map[string]string)
    76  var mountpointLock sync.RWMutex
    77  
    78  var providers = []common.LinuxVolumeQuotaProvider{
    79  	&common.VolumeProvider{},
    80  }
    81  
    82  // Separate the innards for ease of testing
    83  func detectBackingDevInternal(mountpoint string, mounts string) (string, error) {
    84  	file, err := os.Open(mounts)
    85  	if err != nil {
    86  		return "", err
    87  	}
    88  	defer file.Close()
    89  	scanner := bufio.NewScanner(file)
    90  	for scanner.Scan() {
    91  		match := common.MountParseRegexp.FindStringSubmatch(scanner.Text())
    92  		if match != nil {
    93  			device := match[1]
    94  			mount := match[2]
    95  			if mount == mountpoint {
    96  				return device, nil
    97  			}
    98  		}
    99  	}
   100  	return "", fmt.Errorf("couldn't find backing device for %s", mountpoint)
   101  }
   102  
   103  // detectBackingDev assumes that the mount point provided is valid
   104  func detectBackingDev(_ mount.Interface, mountpoint string) (string, error) {
   105  	return detectBackingDevInternal(mountpoint, common.MountsFile)
   106  }
   107  
   108  func clearBackingDev(path string) {
   109  	backingDevLock.Lock()
   110  	defer backingDevLock.Unlock()
   111  	delete(backingDevMap, path)
   112  }
   113  
   114  // Assumes that the path has been fully canonicalized
   115  // Breaking this up helps with testing
   116  func detectMountpointInternal(m mount.Interface, path string) (string, error) {
   117  	for path != "" && path != "/" {
   118  		// per k8s.io/mount-utils/mount_linux this detects all but
   119  		// a bind mount from one part of a mount to another.
   120  		// For our purposes that's fine; we simply want the "true"
   121  		// mount point
   122  		//
   123  		// IsNotMountPoint proved much more troublesome; it actually
   124  		// scans the mounts, and when a lot of mount/unmount
   125  		// activity takes place, it is not able to get a consistent
   126  		// view of /proc/self/mounts, causing it to time out and
   127  		// report incorrectly.
   128  		isNotMount, err := m.IsLikelyNotMountPoint(path)
   129  		if err != nil {
   130  			return "/", err
   131  		}
   132  		if !isNotMount {
   133  			return path, nil
   134  		}
   135  		path = filepath.Dir(path)
   136  	}
   137  	return "/", nil
   138  }
   139  
   140  func detectMountpoint(m mount.Interface, path string) (string, error) {
   141  	xpath, err := filepath.Abs(path)
   142  	if err != nil {
   143  		return "/", err
   144  	}
   145  	xpath, err = filepath.EvalSymlinks(xpath)
   146  	if err != nil {
   147  		return "/", err
   148  	}
   149  	if xpath, err = detectMountpointInternal(m, xpath); err == nil {
   150  		return xpath, nil
   151  	}
   152  	return "/", err
   153  }
   154  
   155  func clearMountpoint(path string) {
   156  	mountpointLock.Lock()
   157  	defer mountpointLock.Unlock()
   158  	delete(mountpointMap, path)
   159  }
   160  
   161  // getFSInfo Returns mountpoint and backing device
   162  // getFSInfo should cache the mountpoint and backing device for the
   163  // path.
   164  func getFSInfo(m mount.Interface, path string) (string, string, error) {
   165  	mountpointLock.Lock()
   166  	defer mountpointLock.Unlock()
   167  
   168  	backingDevLock.Lock()
   169  	defer backingDevLock.Unlock()
   170  
   171  	var err error
   172  
   173  	mountpoint, okMountpoint := mountpointMap[path]
   174  	if !okMountpoint {
   175  		mountpoint, err = detectMountpoint(m, path)
   176  		if err != nil {
   177  			return "", "", fmt.Errorf("cannot determine mountpoint for %s: %v", path, err)
   178  		}
   179  	}
   180  
   181  	backingDev, okBackingDev := backingDevMap[path]
   182  	if !okBackingDev {
   183  		backingDev, err = detectBackingDev(m, mountpoint)
   184  		if err != nil {
   185  			return "", "", fmt.Errorf("cannot determine backing device for %s: %v", path, err)
   186  		}
   187  	}
   188  	mountpointMap[path] = mountpoint
   189  	backingDevMap[path] = backingDev
   190  	return mountpoint, backingDev, nil
   191  }
   192  
   193  func clearFSInfo(path string) {
   194  	clearMountpoint(path)
   195  	clearBackingDev(path)
   196  }
   197  
   198  func getApplier(path string) common.LinuxVolumeQuotaApplier {
   199  	dirApplierLock.Lock()
   200  	defer dirApplierLock.Unlock()
   201  	return dirApplierMap[path]
   202  }
   203  
   204  func setApplier(path string, applier common.LinuxVolumeQuotaApplier) {
   205  	dirApplierLock.Lock()
   206  	defer dirApplierLock.Unlock()
   207  	dirApplierMap[path] = applier
   208  }
   209  
   210  func clearApplier(path string) {
   211  	dirApplierLock.Lock()
   212  	defer dirApplierLock.Unlock()
   213  	delete(dirApplierMap, path)
   214  }
   215  
   216  func setQuotaOnDir(path string, id common.QuotaID, bytes int64) error {
   217  	return getApplier(path).SetQuotaOnDir(path, id, bytes)
   218  }
   219  
   220  func GetQuotaOnDir(m mount.Interface, path string) (common.QuotaID, error) {
   221  	_, _, err := getFSInfo(m, path)
   222  	if err != nil {
   223  		return common.BadQuotaID, err
   224  	}
   225  	return getApplier(path).GetQuotaOnDir(path)
   226  }
   227  
   228  func clearQuotaOnDir(m mount.Interface, path string) error {
   229  	// Since we may be called without path being in the map,
   230  	// we explicitly have to check in this case.
   231  	klog.V(4).Infof("clearQuotaOnDir %s", path)
   232  	supportsQuotas, err := SupportsQuotas(m, path)
   233  	if err != nil {
   234  		// Log-and-continue instead of returning an error for now
   235  		// due to unspecified backwards compatibility concerns (a subject to revise)
   236  		klog.V(3).Infof("Attempt to check for quota support failed: %v", err)
   237  	}
   238  	if !supportsQuotas {
   239  		return nil
   240  	}
   241  	projid, err := GetQuotaOnDir(m, path)
   242  	if err == nil && projid != common.BadQuotaID {
   243  		// This means that we have a quota on the directory but
   244  		// we can't clear it.  That's not good.
   245  		err = setQuotaOnDir(path, projid, 0)
   246  		if err != nil {
   247  			klog.V(3).Infof("Attempt to clear quota failed: %v", err)
   248  		}
   249  		// Even if clearing the quota failed, we still need to
   250  		// try to remove the project ID, or that may be left dangling.
   251  		err1 := removeProjectID(path, projid)
   252  		if err1 != nil {
   253  			klog.V(3).Infof("Attempt to remove quota ID from system files failed: %v", err1)
   254  		}
   255  		clearFSInfo(path)
   256  		if err != nil {
   257  			return err
   258  		}
   259  		return err1
   260  	}
   261  	// If we couldn't get a quota, that's fine -- there may
   262  	// never have been one, and we have no way to know otherwise
   263  	klog.V(3).Infof("clearQuotaOnDir fails %v", err)
   264  	return nil
   265  }
   266  
   267  // SupportsQuotas -- Does the path support quotas
   268  // Cache the applier for paths that support quotas.  For paths that don't,
   269  // don't cache the result because nothing will clean it up.
   270  // However, do cache the device->applier map; the number of devices
   271  // is bounded.
   272  func SupportsQuotas(m mount.Interface, path string) (bool, error) {
   273  	if !enabledQuotasForMonitoring() {
   274  		klog.V(3).Info("SupportsQuotas called, but quotas disabled")
   275  		return false, nil
   276  	}
   277  	supportsQuotasLock.Lock()
   278  	defer supportsQuotasLock.Unlock()
   279  	if supportsQuotas, ok := supportsQuotasMap[path]; ok {
   280  		return supportsQuotas, nil
   281  	}
   282  	mount, dev, err := getFSInfo(m, path)
   283  	if err != nil {
   284  		return false, err
   285  	}
   286  	// Do we know about this device?
   287  	applier, ok := devApplierMap[mount]
   288  	if !ok {
   289  		for _, provider := range providers {
   290  			if applier = provider.GetQuotaApplier(mount, dev); applier != nil {
   291  				devApplierMap[mount] = applier
   292  				break
   293  			}
   294  		}
   295  	}
   296  	if applier != nil {
   297  		supportsQuotasMap[path] = true
   298  		setApplier(path, applier)
   299  		return true, nil
   300  	}
   301  	delete(backingDevMap, path)
   302  	delete(mountpointMap, path)
   303  	return false, nil
   304  }
   305  
   306  // AssignQuota -- assign a quota to the specified directory.
   307  // AssignQuota chooses the quota ID based on the pod UID and path.
   308  // If the pod UID is identical to another one known, it may (but presently
   309  // doesn't) choose the same quota ID as other volumes in the pod.
   310  func AssignQuota(m mount.Interface, path string, poduid types.UID, bytes *resource.Quantity) error { //nolint:staticcheck
   311  	if bytes == nil {
   312  		return fmt.Errorf("attempting to assign null quota to %s", path)
   313  	}
   314  	ibytes := bytes.Value()
   315  	if ok, err := SupportsQuotas(m, path); !ok {
   316  		return fmt.Errorf("quotas not supported on %s: %v", path, err)
   317  	}
   318  	quotaLock.Lock()
   319  	defer quotaLock.Unlock()
   320  	// Current policy is to set individual quotas on each volume,
   321  	// for each new volume we generate a random UUID and we use that as
   322  	// the internal pod uid.
   323  	// From fsquota point of view each volume is attached to a
   324  	// single unique pod.
   325  	// If we decide later that we want to assign one quota for all
   326  	// volumes in a pod, we can simply use poduid parameter directly
   327  	// If and when we decide permanently that we're going to adopt
   328  	// one quota per volume, we can rip all of the pod code out.
   329  	externalPodUid := poduid
   330  	internalPodUid, ok := dirPodMap[path]
   331  	if ok {
   332  		if podUidMap[internalPodUid] != externalPodUid {
   333  			return fmt.Errorf("requesting quota on existing directory %s but different pod %s %s", path, podUidMap[internalPodUid], externalPodUid)
   334  		}
   335  	} else {
   336  		internalPodUid = types.UID(uuid.NewUUID())
   337  	}
   338  	oid, ok := podQuotaMap[internalPodUid]
   339  	if ok {
   340  		if quotaSizeMap[oid] != ibytes {
   341  			return fmt.Errorf("requesting quota of different size: old %v new %v", quotaSizeMap[oid], bytes)
   342  		}
   343  		if _, ok := dirPodMap[path]; ok {
   344  			return nil
   345  		}
   346  	} else {
   347  		oid = common.BadQuotaID
   348  	}
   349  	id, err := createProjectID(path, oid)
   350  	if err == nil {
   351  		if oid != common.BadQuotaID && oid != id {
   352  			return fmt.Errorf("attempt to reassign quota %v to %v", oid, id)
   353  		}
   354  		// When enforcing quotas are enabled, we'll condition this
   355  		// on their being disabled also.
   356  		fsbytes := ibytes
   357  		if fsbytes > 0 {
   358  			fsbytes = -1
   359  		}
   360  		if err = setQuotaOnDir(path, id, fsbytes); err == nil {
   361  			quotaPodMap[id] = internalPodUid
   362  			quotaSizeMap[id] = ibytes
   363  			podQuotaMap[internalPodUid] = id
   364  			dirQuotaMap[path] = id
   365  			dirPodMap[path] = internalPodUid
   366  			podUidMap[internalPodUid] = externalPodUid
   367  			podDirCountMap[internalPodUid]++
   368  			klog.V(4).Infof("Assigning quota ID %d (request limit %d, actual limit %d) to %s", id, ibytes, fsbytes, path)
   369  			return nil
   370  		}
   371  		removeProjectID(path, id)
   372  	}
   373  	return fmt.Errorf("assign quota FAILED %v", err)
   374  }
   375  
   376  // GetConsumption -- retrieve the consumption (in bytes) of the directory
   377  func GetConsumption(path string) (*resource.Quantity, error) {
   378  	// Note that we actually need to hold the lock at least through
   379  	// running the quota command, so it can't get recycled behind our back
   380  	quotaLock.Lock()
   381  	defer quotaLock.Unlock()
   382  	applier := getApplier(path)
   383  	// No applier means directory is not under quota management
   384  	if applier == nil {
   385  		return nil, nil
   386  	}
   387  	ibytes, err := applier.GetConsumption(path, dirQuotaMap[path])
   388  	if err != nil {
   389  		return nil, err
   390  	}
   391  	return resource.NewQuantity(ibytes, resource.DecimalSI), nil
   392  }
   393  
   394  // GetInodes -- retrieve the number of inodes in use under the directory
   395  func GetInodes(path string) (*resource.Quantity, error) {
   396  	// Note that we actually need to hold the lock at least through
   397  	// running the quota command, so it can't get recycled behind our back
   398  	quotaLock.Lock()
   399  	defer quotaLock.Unlock()
   400  	applier := getApplier(path)
   401  	// No applier means directory is not under quota management
   402  	if applier == nil {
   403  		return nil, nil
   404  	}
   405  	inodes, err := applier.GetInodes(path, dirQuotaMap[path])
   406  	if err != nil {
   407  		return nil, err
   408  	}
   409  	return resource.NewQuantity(inodes, resource.DecimalSI), nil
   410  }
   411  
   412  // ClearQuota -- remove the quota assigned to a directory
   413  func ClearQuota(m mount.Interface, path string) error {
   414  	klog.V(3).Infof("ClearQuota %s", path)
   415  	if !enabledQuotasForMonitoring() {
   416  		return fmt.Errorf("clearQuota called, but quotas disabled")
   417  	}
   418  	quotaLock.Lock()
   419  	defer quotaLock.Unlock()
   420  	poduid, ok := dirPodMap[path]
   421  	if !ok {
   422  		// Nothing in the map either means that there was no
   423  		// quota to begin with or that we're clearing a
   424  		// stale directory, so if we find a quota, just remove it.
   425  		// The process of clearing the quota requires that an applier
   426  		// be found, which needs to be cleaned up.
   427  		defer delete(supportsQuotasMap, path)
   428  		defer clearApplier(path)
   429  		return clearQuotaOnDir(m, path)
   430  	}
   431  	_, ok = podQuotaMap[poduid]
   432  	if !ok {
   433  		return fmt.Errorf("clearQuota: No quota available for %s", path)
   434  	}
   435  	projid, err := GetQuotaOnDir(m, path)
   436  	if err != nil {
   437  		// Log-and-continue instead of returning an error for now
   438  		// due to unspecified backwards compatibility concerns (a subject to revise)
   439  		klog.V(3).Infof("Attempt to check quota ID %v on dir %s failed: %v", dirQuotaMap[path], path, err)
   440  	}
   441  	if projid != dirQuotaMap[path] {
   442  		return fmt.Errorf("expected quota ID %v on dir %s does not match actual %v", dirQuotaMap[path], path, projid)
   443  	}
   444  	count, ok := podDirCountMap[poduid]
   445  	if count <= 1 || !ok {
   446  		err = clearQuotaOnDir(m, path)
   447  		// This error should be noted; we still need to clean up
   448  		// and otherwise handle in the same way.
   449  		if err != nil {
   450  			klog.V(3).Infof("Unable to clear quota %v %s: %v", dirQuotaMap[path], path, err)
   451  		}
   452  		delete(quotaSizeMap, podQuotaMap[poduid])
   453  		delete(quotaPodMap, podQuotaMap[poduid])
   454  		delete(podDirCountMap, poduid)
   455  		delete(podQuotaMap, poduid)
   456  		delete(podUidMap, poduid)
   457  	} else {
   458  		err = removeProjectID(path, projid)
   459  		podDirCountMap[poduid]--
   460  		klog.V(4).Infof("Not clearing quota for pod %s; still %v dirs outstanding", poduid, podDirCountMap[poduid])
   461  	}
   462  	delete(dirPodMap, path)
   463  	delete(dirQuotaMap, path)
   464  	delete(supportsQuotasMap, path)
   465  	clearApplier(path)
   466  	if err != nil {
   467  		return fmt.Errorf("unable to clear quota for %s: %v", path, err)
   468  	}
   469  	return nil
   470  }
   471
View as plain text