1#!/usr/bin/env bash
2
3# Copyright 2015 The Kubernetes Authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17# This script grabs profiles from running components.
18# Usage: `hack/grab-profiles.sh`.
19
20set -o errexit
21set -o nounset
22set -o pipefail
23
24function grab_profiles_from_component {
25 local requested_profiles=$1
26 local mem_pprof_flags=$2
27 local binary=$3
28 local tunnel_port=$4
29 local path=$5
30 local output_prefix=$6
31 local timestamp=$7
32
33 echo "binary: $binary"
34
35 for profile in ${requested_profiles}; do
36 case ${profile} in
37 cpu)
38 go tool pprof "-pdf" "${binary}" "http://localhost:${tunnel_port}${path}/debug/pprof/profile" > "${output_prefix}-${profile}-profile-${timestamp}.pdf"
39 ;;
40 mem)
41 # There are different kinds of memory profiles that are available that
42 # had to be grabbed separately: --inuse-space, --inuse-objects,
43 # --alloc-space, --alloc-objects. We need to iterate over all requested
44 # kinds.
45 for flag in ${mem_pprof_flags}; do
46 go tool pprof "-${flag}" "-pdf" "${binary}" "http://localhost:${tunnel_port}${path}/debug/pprof/heap" > "${output_prefix}-${profile}-${flag}-profile-${timestamp}.pdf"
47 done
48 ;;
49 esac
50 done
51}
52
53KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
54source "${KUBE_ROOT}/hack/lib/init.sh"
55
56server_addr=""
57kubelet_addresses=""
58kubelet_binary=""
59master_binary=""
60scheduler_binary=""
61scheduler_port="10251"
62controller_manager_port="10252"
63controller_manager_binary=""
64requested_profiles=""
65mem_pprof_flags=""
66profile_components=""
67output_dir="."
68tunnel_port="${tunnel_port:-1234}"
69
70if ! args=$(getopt -o s:mho:k:c -l server:,master,heapster,output:,kubelet:,scheduler,controller-manager,help,inuse-space,inuse-objects,alloc-space,alloc-objects,cpu,kubelet-binary:,master-binary:,scheduler-binary:,controller-manager-binary:,scheduler-port:,controller-manager-port: -- "$@"); then
71 >&2 echo "Error in getopt"
72 exit 1
73fi
74
75HEAPSTER_VERSION="v0.18.2"
76MASTER_PPROF_PATH=""
77HEAPSTER_PPROF_PATH="/api/v1/namespaces/kube-system/services/monitoring-heapster/proxy"
78KUBELET_PPROF_PATH_PREFIX="/api/v1/proxy/nodes"
79SCHEDULER_PPROF_PATH_PREFIX="/api/v1/namespaces/kube-system/pods/kube-scheduler/proxy"
80CONTROLLER_MANAGER_PPROF_PATH_PREFIX="/api/v1/namespaces/kube-system/pods/kube-controller-manager/proxy"
81
82eval set -- "${args}"
83
84while true; do
85 case $1 in
86 -s|--server)
87 shift
88 if [ -z "$1" ]; then
89 >&2 echo "empty argument to --server flag"
90 exit 1
91 fi
92 server_addr=$1
93 shift
94 ;;
95 -m|--master)
96 shift
97 profile_components="master ${profile_components}"
98 ;;
99 --master-binary)
100 shift
101 if [ -z "$1" ]; then
102 >&2 echo "empty argument to --master-binary flag"
103 exit 1
104 fi
105 master_binary=$1
106 shift
107 ;;
108 -h|--heapster)
109 shift
110 profile_components="heapster ${profile_components}"
111 ;;
112 -k|--kubelet)
113 shift
114 profile_components="kubelet ${profile_components}"
115 if [ -z "$1" ]; then
116 >&2 echo "empty argument to --kubelet flag"
117 exit 1
118 fi
119 kubelet_addresses="$1 $kubelet_addresses"
120 shift
121 ;;
122 --kubelet-binary)
123 shift
124 if [ -z "$1" ]; then
125 >&2 echo "empty argument to --kubelet-binary flag"
126 exit 1
127 fi
128 kubelet_binary=$1
129 shift
130 ;;
131 --scheduler)
132 shift
133 profile_components="scheduler ${profile_components}"
134 ;;
135 --scheduler-binary)
136 shift
137 if [ -z "$1" ]; then
138 >&2 echo "empty argument to --scheduler-binary flag"
139 exit 1
140 fi
141 scheduler_binary=$1
142 shift
143 ;;
144 --scheduler-port)
145 shift
146 if [ -z "$1" ]; then
147 >&2 echo "empty argument to --scheduler-port flag"
148 exit 1
149 fi
150 scheduler_port=$1
151 shift
152 ;;
153 -c|--controller-manager)
154 shift
155 profile_components="controller-manager ${profile_components}"
156 ;;
157 --controller-manager-binary)
158 shift
159 if [ -z "$1" ]; then
160 >&2 echo "empty argument to --controller-manager-binary flag"
161 exit 1
162 fi
163 controller_manager_binary=$1
164 shift
165 ;;
166 --controller-manager-port)
167 shift
168 if [ -z "$1" ]; then
169 >&2 echo "empty argument to --controller-manager-port flag"
170 exit 1
171 fi
172 controller_manager_port=$1
173 shift
174 ;;
175 -o|--output)
176 shift
177 if [ -z "$1" ]; then
178 >&2 echo "empty argument to --output flag"
179 exit 1
180 fi
181 output_dir=$1
182 shift
183 ;;
184 --inuse-space)
185 shift
186 requested_profiles="mem ${requested_profiles}"
187 mem_pprof_flags="inuse_space ${mem_pprof_flags}"
188 ;;
189 --inuse-objects)
190 shift
191 requested_profiles="mem ${requested_profiles}"
192 mem_pprof_flags="inuse_objects ${mem_pprof_flags}"
193 ;;
194 --alloc-space)
195 shift
196 requested_profiles="mem ${requested_profiles}"
197 mem_pprof_flags="alloc_space ${mem_pprof_flags}"
198 ;;
199 --alloc-objects)
200 shift
201 requested_profiles="mem ${requested_profiles}"
202 mem_pprof_flags="alloc_objects ${mem_pprof_flags}"
203 ;;
204 --cpu)
205 shift
206 requested_profiles="cpu ${requested_profiles}"
207 ;;
208 --help)
209 shift
210 echo "Recognized options:
211 -o/--output,
212 -s/--server,
213 -m/--master,
214 -h/--heapster,
215 --inuse-space,
216 --inuse-objects,
217 --alloc-space,
218 --alloc-objects,
219 --cpu,
220 --help"
221 exit 0
222 ;;
223 --)
224 shift
225 break;
226 ;;
227 esac
228done
229
230if [[ -z "${server_addr}" ]]; then
231 >&2 echo "Server flag is required"
232 exit 1
233fi
234
235if [[ -z "${profile_components}" ]]; then
236 >&2 echo "Choose at least one component to profile"
237 exit 1
238fi
239
240if [[ -z "${requested_profiles}" ]]; then
241 >&2 echo "Choose at least one profiling option"
242 exit 1
243fi
244
245gcloud compute ssh "${server_addr}" --ssh-flag=-nN --ssh-flag=-L"${tunnel_port}":localhost:8080 &
246
247echo "Waiting for tunnel to be created..."
248kube::util::wait_for_url http://localhost:"${tunnel_port}"/healthz
249
250SSH_PID=$(pgrep -f "/usr/bin/ssh.*${tunnel_port}:localhost:8080")
251kube::util::trap_add "kill $SSH_PID" EXIT
252kube::util::trap_add "kill $SSH_PID" SIGTERM
253
254requested_profiles=$(echo "${requested_profiles}" | xargs -n1 | LC_ALL=C sort -u | xargs)
255profile_components=$(echo "${profile_components}" | xargs -n1 | LC_ALL=C sort -u | xargs)
256kubelet_addresses=$(echo "${kubelet_addresses}" | xargs -n1 | LC_ALL=C sort -u | xargs)
257echo "requested profiles: ${requested_profiles}"
258echo "flags for heap profile: ${mem_pprof_flags}"
259
260timestamp=$(date +%Y%m%d%H%M%S)
261binary=""
262
263for component in ${profile_components}; do
264 case ${component} in
265 master)
266 path=${MASTER_PPROF_PATH}
267 binary=${master_binary}
268 ;;
269 controller-manager)
270 path="${CONTROLLER_MANAGER_PPROF_PATH_PREFIX}-${server_addr}:${controller_manager_port}"
271 binary=${controller_manager_binary}
272 ;;
273 scheduler)
274 path="${SCHEDULER_PPROF_PATH_PREFIX}-${server_addr}:${scheduler_port}"
275 binary=${scheduler_binary}
276 ;;
277 heapster)
278 rm heapster
279 wget https://github.com/kubernetes/heapster/releases/download/${HEAPSTER_VERSION}/heapster
280 kube::util::trap_add 'rm -f heapster' EXIT
281 kube::util::trap_add 'rm -f heapster' SIGTERM
282 binary=heapster
283 path=${HEAPSTER_PPROF_PATH}
284 ;;
285 kubelet)
286 path="${KUBELET_PPROF_PATH_PREFIX}"
287 if [[ -z "${kubelet_binary}" ]]; then
288 binary="${KUBE_ROOT}/_output/local/bin/linux/amd64/kubelet"
289 else
290 binary=${kubelet_binary}
291 fi
292 ;;
293 esac
294
295 if [[ "${component}" == "kubelet" ]]; then
296 for node in ${kubelet_addresses//[,;]/' '}; do
297 grab_profiles_from_component "${requested_profiles}" "${mem_pprof_flags}" "${binary}" "${tunnel_port}" "${path}/${node}/proxy" "${output_dir}/${component}" "${timestamp}"
298 done
299 else
300 grab_profiles_from_component "${requested_profiles}" "${mem_pprof_flags}" "${binary}" "${tunnel_port}" "${path}" "${output_dir}/${component}" "${timestamp}"
301 fi
302done
View as plain text