1package kube
2
3import yaml656e63 "encoding/yaml"
4
5configMap: prometheus: {
6 apiVersion: "v1"
7 kind: "ConfigMap"
8 data: {
9 "alert.rules": yaml656e63.Marshal(_cue_alert_rules)
10 let _cue_alert_rules = {
11 groups: [{
12 name: "rules.yaml"
13 rules: [{
14 alert: "InstanceDown"
15 expr: "up == 0"
16 for: "30s"
17 labels: severity: "page"
18 annotations: {
19 description: "{{$labels.app}} of job {{ $labels.job }} has been down for more than 30 seconds."
20
21 summary: "Instance {{$labels.app}} down"
22 }
23 }, {
24 alert: "InsufficientPeers"
25 expr: "count(up{job=\"etcd\"} == 0) > (count(up{job=\"etcd\"}) / 2 - 1)"
26 for: "3m"
27 labels: severity: "page"
28 annotations: {
29 description: "If one more etcd peer goes down the cluster will be unavailable"
30 summary: "etcd cluster small"
31 }
32 }, {
33 alert: "EtcdNoMaster"
34 expr: "sum(etcd_server_has_leader{app=\"etcd\"}) == 0"
35 for: "1s"
36 labels: severity: "page"
37 annotations: summary: "No ETCD master elected."
38 }, {
39 alert: "PodRestart"
40 expr: "(max_over_time(pod_container_status_restarts_total[5m]) - min_over_time(pod_container_status_restarts_total[5m])) > 2"
41 for: "1m"
42 labels: severity: "page"
43 annotations: {
44 description: "{{$labels.app}} {{ $labels.container }} resturted {{ $value }} times in 5m."
45 summary: "Pod for {{$labels.container}} restarts too often"
46 }
47 }]
48 }]
49 }
50
51 "prometheus.yml": yaml656e63.Marshal(_cue_prometheus_yml)
52 let _cue_prometheus_yml = {
53 global: scrape_interval: "15s"
54 rule_files: ["/etc/prometheus/alert.rules"]
55 alerting: alertmanagers: [{
56 scheme: "http"
57 static_configs: [{
58 targets: ["alertmanager:9093"]
59 }]
60 }]
61 scrape_configs: [{
62 job_name: "kubernetes-apiservers"
63
64 kubernetes_sd_configs: [{
65 role: "endpoints"
66 }]
67
68 // Default to scraping over https. If required, just disable this or change to
69 // `http`.
70 scheme: "https"
71
72 // This TLS & bearer token file config is used to connect to the actual scrape
73 // endpoints for cluster components. This is separate to discovery auth
74 // configuration because discovery & scraping are two separate concerns in
75 // Prometheus. The discovery auth config is automatic if Prometheus runs inside
76 // the cluster. Otherwise, more config options have to be provided within the
77 // <kubernetes_sd_config>.
78 tls_config: {
79 ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
80 }
81 // If your node certificates are self-signed or use a different CA to the
82 // master CA, then disable certificate verification below. Note that
83 // certificate verification is an integral part of a secure infrastructure
84 // so this should only be disabled in a controlled environment. You can
85 // disable certificate verification by uncommenting the line below.
86 //
87 // insecure_skip_verify: true
88 bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token"
89
90 // Keep only the default/kubernetes service endpoints for the https port. This
91 // will add targets for each API server which Kubernetes adds an endpoint to
92 // the default/kubernetes service.
93 relabel_configs: [{
94 source_labels: ["__meta_kubernetes_namespace", "__meta_kubernetes_service_name", "__meta_kubernetes_endpoint_port_name"]
95 action: "keep"
96 regex: "default;kubernetes;https"
97 }]
98 }, {
99 // Scrape config for nodes (kubelet).
100 //
101 // Rather than connecting directly to the node, the scrape is proxied though the
102 // Kubernetes apiserver. This means it will work if Prometheus is running out of
103 // cluster, or can't connect to nodes for some other reason (e.g. because of
104 // firewalling).
105 job_name: "kubernetes-nodes"
106
107 // Default to scraping over https. If required, just disable this or change to
108 // `http`.
109 scheme: "https"
110
111 // This TLS & bearer token file config is used to connect to the actual scrape
112 // endpoints for cluster components. This is separate to discovery auth
113 // configuration because discovery & scraping are two separate concerns in
114 // Prometheus. The discovery auth config is automatic if Prometheus runs inside
115 // the cluster. Otherwise, more config options have to be provided within the
116 // <kubernetes_sd_config>.
117 tls_config: {
118 ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
119 }
120 bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token"
121
122 kubernetes_sd_configs: [{
123 role: "node"
124 }]
125
126 relabel_configs: [{
127 action: "labelmap"
128 regex: "__meta_kubernetes_node_label_(.+)"
129 }, {
130 target_label: "__address__"
131 replacement: "kubernetes.default.svc:443"
132 }, {
133 source_labels: ["__meta_kubernetes_node_name"]
134 regex: "(.+)"
135 target_label: "__metrics_path__"
136 replacement: "/api/v1/nodes/${1}/proxy/metrics"
137 }]
138 }, {
139 // Scrape config for Kubelet cAdvisor.
140 //
141 // This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics
142 // (those whose names begin with 'container_') have been removed from the
143 // Kubelet metrics endpoint. This job scrapes the cAdvisor endpoint to
144 // retrieve those metrics.
145 //
146 // In Kubernetes 1.7.0-1.7.2, these metrics are only exposed on the cAdvisor
147 // HTTP endpoint; use "replacement: /api/v1/nodes/${1}:4194/proxy/metrics"
148 // in that case (and ensure cAdvisor's HTTP server hasn't been disabled with
149 // the --cadvisor-port=0 Kubelet flag).
150 //
151 // This job is not necessary and should be removed in Kubernetes 1.6 and
152 // earlier versions, or it will cause the metrics to be scraped twice.
153 job_name: "kubernetes-cadvisor"
154
155 // Default to scraping over https. If required, just disable this or change to
156 // `http`.
157 scheme: "https"
158
159 // This TLS & bearer token file config is used to connect to the actual scrape
160 // endpoints for cluster components. This is separate to discovery auth
161 // configuration because discovery & scraping are two separate concerns in
162 // Prometheus. The discovery auth config is automatic if Prometheus runs inside
163 // the cluster. Otherwise, more config options have to be provided within the
164 // <kubernetes_sd_config>.
165 tls_config: {
166 ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
167 }
168 bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token"
169
170 kubernetes_sd_configs: [{
171 role: "node"
172 }]
173
174 relabel_configs: [{
175 action: "labelmap"
176 regex: "__meta_kubernetes_node_label_(.+)"
177 }, {
178 target_label: "__address__"
179 replacement: "kubernetes.default.svc:443"
180 }, {
181 source_labels: ["__meta_kubernetes_node_name"]
182 regex: "(.+)"
183 target_label: "__metrics_path__"
184 replacement: "/api/v1/nodes/${1}/proxy/metrics/cadvisor"
185 }]
186 }, {
187 // Scrape config for service endpoints.
188 //
189 // The relabeling allows the actual service scrape endpoint to be configured
190 // via the following annotations:
191 //
192 // * `prometheus.io/scrape`: Only scrape services that have a value of `true`
193 // * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need
194 // to set this to `https` & most likely set the `tls_config` of the scrape config.
195 // * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
196 // * `prometheus.io/port`: If the metrics are exposed on a different port to the
197 // service then set this appropriately.
198 job_name: "kubernetes-service-endpoints"
199
200 kubernetes_sd_configs: [{
201 role: "endpoints"
202 }]
203
204 relabel_configs: [{
205 source_labels: ["__meta_kubernetes_service_annotation_prometheus_io_scrape"]
206 action: "keep"
207 regex: true
208 }, {
209 source_labels: ["__meta_kubernetes_service_annotation_prometheus_io_scheme"]
210 action: "replace"
211 target_label: "__scheme__"
212 regex: "(https?)"
213 }, {
214 source_labels: ["__meta_kubernetes_service_annotation_prometheus_io_path"]
215 action: "replace"
216 target_label: "__metrics_path__"
217 regex: "(.+)"
218 }, {
219 source_labels: ["__address__", "__meta_kubernetes_service_annotation_prometheus_io_port"]
220 action: "replace"
221 target_label: "__address__"
222 regex: "([^:]+)(?::\\d+)?;(\\d+)"
223 replacement: "$1:$2"
224 }, {
225 action: "labelmap"
226 regex: "__meta_kubernetes_service_label_(.+)"
227 }, {
228 source_labels: ["__meta_kubernetes_namespace"]
229 action: "replace"
230 target_label: "kubernetes_namespace"
231 }, {
232 source_labels: ["__meta_kubernetes_service_name"]
233 action: "replace"
234 target_label: "kubernetes_name"
235 }]
236 }, {
237 // Example scrape config for probing services via the Blackbox Exporter.
238 //
239 // The relabeling allows the actual service scrape endpoint to be configured
240 // via the following annotations:
241 //
242 // * `prometheus.io/probe`: Only probe services that have a value of `true`
243 job_name: "kubernetes-services"
244
245 metrics_path: "/probe"
246 params: module: ["http_2xx"]
247
248 kubernetes_sd_configs: [{
249 role: "service"
250 }]
251
252 relabel_configs: [{
253 source_labels: ["__meta_kubernetes_service_annotation_prometheus_io_probe"]
254 action: "keep"
255 regex: true
256 }, {
257 source_labels: ["__address__"]
258 target_label: "__param_target"
259 }, {
260 target_label: "__address__"
261 replacement: "blackbox-exporter.example.com:9115"
262 }, {
263 source_labels: ["__param_target"]
264 target_label: "app"
265 }, {
266 action: "labelmap"
267 regex: "__meta_kubernetes_service_label_(.+)"
268 }, {
269 source_labels: ["__meta_kubernetes_namespace"]
270 target_label: "kubernetes_namespace"
271 }, {
272 source_labels: ["__meta_kubernetes_service_name"]
273 target_label: "kubernetes_name"
274 }]
275 }, {
276 // Example scrape config for probing ingresses via the Blackbox Exporter.
277 //
278 // The relabeling allows the actual ingress scrape endpoint to be configured
279 // via the following annotations:
280 //
281 // * `prometheus.io/probe`: Only probe services that have a value of `true`
282 job_name: "kubernetes-ingresses"
283
284 metrics_path: "/probe"
285 params: module: ["http_2xx"]
286
287 kubernetes_sd_configs: [{
288 role: "ingress"
289 }]
290
291 relabel_configs: [{
292 source_labels: ["__meta_kubernetes_ingress_annotation_prometheus_io_probe"]
293 action: "keep"
294 regex: true
295 }, {
296 source_labels: ["__meta_kubernetes_ingress_scheme", "__address__", "__meta_kubernetes_ingress_path"]
297 regex: "(.+);(.+);(.+)"
298 replacement: "${1}://${2}${3}"
299 target_label: "__param_target"
300 }, {
301 target_label: "__address__"
302 replacement: "blackbox-exporter.example.com:9115"
303 }, {
304 source_labels: ["__param_target"]
305 target_label: "app"
306 }, {
307 action: "labelmap"
308 regex: "__meta_kubernetes_ingress_label_(.+)"
309 }, {
310 source_labels: ["__meta_kubernetes_namespace"]
311 target_label: "kubernetes_namespace"
312 }, {
313 source_labels: ["__meta_kubernetes_ingress_name"]
314 target_label: "kubernetes_name"
315 }]
316 }, {
317 // Example scrape config for pods
318 //
319 // The relabeling allows the actual pod scrape endpoint to be configured via the
320 // following annotations:
321 //
322 // * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
323 // * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
324 // * `prometheus.io/port`: Scrape the pod on the indicated port instead of the
325 // pod's declared ports (default is a port-free target if none are declared).
326 job_name: "kubernetes-pods"
327
328 kubernetes_sd_configs: [{
329 role: "pod"
330 }]
331
332 relabel_configs: [{
333 source_labels: ["__meta_kubernetes_pod_annotation_prometheus_io_scrape"]
334 action: "keep"
335 regex: true
336 }, {
337 source_labels: ["__meta_kubernetes_pod_annotation_prometheus_io_path"]
338 action: "replace"
339 target_label: "__metrics_path__"
340 regex: "(.+)"
341 }, {
342 source_labels: ["__address__", "__meta_kubernetes_pod_annotation_prometheus_io_port"]
343 action: "replace"
344 regex: "([^:]+)(?::\\d+)?;(\\d+)"
345 replacement: "$1:$2"
346 target_label: "__address__"
347 }, {
348 action: "labelmap"
349 regex: "__meta_kubernetes_pod_label_(.+)"
350 }, {
351 source_labels: ["__meta_kubernetes_namespace"]
352 action: "replace"
353 target_label: "kubernetes_namespace"
354 }, {
355 source_labels: ["__meta_kubernetes_pod_name"]
356 action: "replace"
357 target_label: "kubernetes_pod_name"
358 }]
359 }]
360 }
361 }
362}
View as plain text