oom_linux.go

Documentation: k8s.io/kubernetes/pkg/util/oom

     1  //go:build linux
     2  // +build linux
     3  
     4  /*
     5  Copyright 2015 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package oom
    21  
    22  import (
    23  	"fmt"
    24  	"os"
    25  	"path"
    26  	"path/filepath"
    27  	"strconv"
    28  	"time"
    29  
    30  	cmutil "k8s.io/kubernetes/pkg/kubelet/cm/util"
    31  
    32  	"k8s.io/klog/v2"
    33  )
    34  
    35  func NewOOMAdjuster() *OOMAdjuster {
    36  	oomAdjuster := &OOMAdjuster{
    37  		pidLister:        getPids,
    38  		ApplyOOMScoreAdj: applyOOMScoreAdj,
    39  	}
    40  	oomAdjuster.ApplyOOMScoreAdjContainer = oomAdjuster.applyOOMScoreAdjContainer
    41  	return oomAdjuster
    42  }
    43  
    44  func getPids(cgroupName string) ([]int, error) {
    45  	return cmutil.GetPids(filepath.Join("/", cgroupName))
    46  }
    47  
    48  // Writes 'value' to /proc/<pid>/oom_score_adj. PID = 0 means self
    49  // Returns os.ErrNotExist if the `pid` does not exist.
    50  func applyOOMScoreAdj(pid int, oomScoreAdj int) error {
    51  	if pid < 0 {
    52  		return fmt.Errorf("invalid PID %d specified for oom_score_adj", pid)
    53  	}
    54  
    55  	var pidStr string
    56  	if pid == 0 {
    57  		pidStr = "self"
    58  	} else {
    59  		pidStr = strconv.Itoa(pid)
    60  	}
    61  
    62  	maxTries := 2
    63  	oomScoreAdjPath := path.Join("/proc", pidStr, "oom_score_adj")
    64  	value := strconv.Itoa(oomScoreAdj)
    65  	klog.V(4).Infof("attempting to set %q to %q", oomScoreAdjPath, value)
    66  	var err error
    67  	for i := 0; i < maxTries; i++ {
    68  		err = os.WriteFile(oomScoreAdjPath, []byte(value), 0700)
    69  		if err != nil {
    70  			if os.IsNotExist(err) {
    71  				klog.V(2).Infof("%q does not exist", oomScoreAdjPath)
    72  				return os.ErrNotExist
    73  			}
    74  
    75  			klog.V(3).Info(err)
    76  			time.Sleep(100 * time.Millisecond)
    77  			continue
    78  		}
    79  		return nil
    80  	}
    81  	if err != nil {
    82  		klog.V(2).Infof("failed to set %q to %q: %v", oomScoreAdjPath, value, err)
    83  	}
    84  	return err
    85  }
    86  
    87  // Writes 'value' to /proc/<pid>/oom_score_adj for all processes in cgroup cgroupName.
    88  // Keeps trying to write until the process list of the cgroup stabilizes, or until maxTries tries.
    89  func (oomAdjuster *OOMAdjuster) applyOOMScoreAdjContainer(cgroupName string, oomScoreAdj, maxTries int) error {
    90  	adjustedProcessSet := make(map[int]bool)
    91  	for i := 0; i < maxTries; i++ {
    92  		continueAdjusting := false
    93  		pidList, err := oomAdjuster.pidLister(cgroupName)
    94  		if err != nil {
    95  			if os.IsNotExist(err) {
    96  				// Nothing to do since the container doesn't exist anymore.
    97  				return os.ErrNotExist
    98  			}
    99  			continueAdjusting = true
   100  			klog.V(10).Infof("Error getting process list for cgroup %s: %+v", cgroupName, err)
   101  		} else if len(pidList) == 0 {
   102  			klog.V(10).Infof("Pid list is empty")
   103  			continueAdjusting = true
   104  		} else {
   105  			for _, pid := range pidList {
   106  				if !adjustedProcessSet[pid] {
   107  					klog.V(10).Infof("pid %d needs to be set", pid)
   108  					if err = oomAdjuster.ApplyOOMScoreAdj(pid, oomScoreAdj); err == nil {
   109  						adjustedProcessSet[pid] = true
   110  					} else if err == os.ErrNotExist {
   111  						continue
   112  					} else {
   113  						klog.V(10).Infof("cannot adjust oom score for pid %d - %v", pid, err)
   114  						continueAdjusting = true
   115  					}
   116  					// Processes can come and go while we try to apply oom score adjust value. So ignore errors here.
   117  				}
   118  			}
   119  		}
   120  		if !continueAdjusting {
   121  			return nil
   122  		}
   123  		// There's a slight race. A process might have forked just before we write its OOM score adjust.
   124  		// The fork might copy the parent process's old OOM score, then this function might execute and
   125  		// update the parent's OOM score, but the forked process id might not be reflected in cgroup.procs
   126  		// for a short amount of time. So this function might return without changing the forked process's
   127  		// OOM score. Very unlikely race, so ignoring this for now.
   128  	}
   129  	return fmt.Errorf("exceeded maxTries, some processes might not have desired OOM score")
   130  }
   131
View as plain text