1 /* 2 Copyright 2014 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package runtime 18 19 import ( 20 "fmt" 21 "net/http" 22 "runtime" 23 "sync" 24 "time" 25 26 "k8s.io/klog/v2" 27 ) 28 29 var ( 30 // ReallyCrash controls the behavior of HandleCrash and defaults to 31 // true. It's exposed so components can optionally set to false 32 // to restore prior behavior. This flag is mostly used for tests to validate 33 // crash conditions. 34 ReallyCrash = true 35 ) 36 37 // PanicHandlers is a list of functions which will be invoked when a panic happens. 38 var PanicHandlers = []func(interface{}){logPanic} 39 40 // HandleCrash simply catches a crash and logs an error. Meant to be called via 41 // defer. Additional context-specific handlers can be provided, and will be 42 // called in case of panic. HandleCrash actually crashes, after calling the 43 // handlers and logging the panic message. 44 // 45 // E.g., you can provide one or more additional handlers for something like shutting down go routines gracefully. 46 func HandleCrash(additionalHandlers ...func(interface{})) { 47 if r := recover(); r != nil { 48 for _, fn := range PanicHandlers { 49 fn(r) 50 } 51 for _, fn := range additionalHandlers { 52 fn(r) 53 } 54 if ReallyCrash { 55 // Actually proceed to panic. 56 panic(r) 57 } 58 } 59 } 60 61 // logPanic logs the caller tree when a panic occurs (except in the special case of http.ErrAbortHandler). 62 func logPanic(r interface{}) { 63 if r == http.ErrAbortHandler { 64 // honor the http.ErrAbortHandler sentinel panic value: 65 // ErrAbortHandler is a sentinel panic value to abort a handler. 66 // While any panic from ServeHTTP aborts the response to the client, 67 // panicking with ErrAbortHandler also suppresses logging of a stack trace to the server's error log. 68 return 69 } 70 71 // Same as stdlib http server code. Manually allocate stack trace buffer size 72 // to prevent excessively large logs 73 const size = 64 << 10 74 stacktrace := make([]byte, size) 75 stacktrace = stacktrace[:runtime.Stack(stacktrace, false)] 76 if _, ok := r.(string); ok { 77 klog.Errorf("Observed a panic: %s\n%s", r, stacktrace) 78 } else { 79 klog.Errorf("Observed a panic: %#v (%v)\n%s", r, r, stacktrace) 80 } 81 } 82 83 // ErrorHandlers is a list of functions which will be invoked when a nonreturnable 84 // error occurs. 85 // TODO(lavalamp): for testability, this and the below HandleError function 86 // should be packaged up into a testable and reusable object. 87 var ErrorHandlers = []func(error){ 88 logError, 89 (&rudimentaryErrorBackoff{ 90 lastErrorTime: time.Now(), 91 // 1ms was the number folks were able to stomach as a global rate limit. 92 // If you need to log errors more than 1000 times a second you 93 // should probably consider fixing your code instead. :) 94 minPeriod: time.Millisecond, 95 }).OnError, 96 } 97 98 // HandlerError is a method to invoke when a non-user facing piece of code cannot 99 // return an error and needs to indicate it has been ignored. Invoking this method 100 // is preferable to logging the error - the default behavior is to log but the 101 // errors may be sent to a remote server for analysis. 102 func HandleError(err error) { 103 // this is sometimes called with a nil error. We probably shouldn't fail and should do nothing instead 104 if err == nil { 105 return 106 } 107 108 for _, fn := range ErrorHandlers { 109 fn(err) 110 } 111 } 112 113 // logError prints an error with the call stack of the location it was reported 114 func logError(err error) { 115 klog.ErrorDepth(2, err) 116 } 117 118 type rudimentaryErrorBackoff struct { 119 minPeriod time.Duration // immutable 120 // TODO(lavalamp): use the clock for testability. Need to move that 121 // package for that to be accessible here. 122 lastErrorTimeLock sync.Mutex 123 lastErrorTime time.Time 124 } 125 126 // OnError will block if it is called more often than the embedded period time. 127 // This will prevent overly tight hot error loops. 128 func (r *rudimentaryErrorBackoff) OnError(error) { 129 now := time.Now() // start the timer before acquiring the lock 130 r.lastErrorTimeLock.Lock() 131 d := now.Sub(r.lastErrorTime) 132 r.lastErrorTime = time.Now() 133 r.lastErrorTimeLock.Unlock() 134 135 // Do not sleep with the lock held because that causes all callers of HandleError to block. 136 // We only want the current goroutine to block. 137 // A negative or zero duration causes time.Sleep to return immediately. 138 // If the time moves backwards for any reason, do nothing. 139 time.Sleep(r.minPeriod - d) 140 } 141 142 // GetCaller returns the caller of the function that calls it. 143 func GetCaller() string { 144 var pc [1]uintptr 145 runtime.Callers(3, pc[:]) 146 f := runtime.FuncForPC(pc[0]) 147 if f == nil { 148 return "Unable to find caller" 149 } 150 return f.Name() 151 } 152 153 // RecoverFromPanic replaces the specified error with an error containing the 154 // original error, and the call tree when a panic occurs. This enables error 155 // handlers to handle errors and panics the same way. 156 func RecoverFromPanic(err *error) { 157 if r := recover(); r != nil { 158 // Same as stdlib http server code. Manually allocate stack trace buffer size 159 // to prevent excessively large logs 160 const size = 64 << 10 161 stacktrace := make([]byte, size) 162 stacktrace = stacktrace[:runtime.Stack(stacktrace, false)] 163 164 *err = fmt.Errorf( 165 "recovered from panic %q. (err=%v) Call stack:\n%s", 166 r, 167 *err, 168 stacktrace) 169 } 170 } 171 172 // Must panics on non-nil errors. Useful to handling programmer level errors. 173 func Must(err error) { 174 if err != nil { 175 panic(err) 176 } 177 } 178