1 package metrics
2
3 import (
4 "strings"
5 "time"
6
7 "github.com/prometheus/client_golang/prometheus"
8 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9 ctrl "sigs.k8s.io/controller-runtime"
10 "sigs.k8s.io/controller-runtime/pkg/client"
11
12 "edge-infra.dev/pkg/k8s/runtime/controller/metrics"
13 v1etcd "edge-infra.dev/pkg/sds/etcd/operator/apis/etcdmember/v1"
14 )
15
16 const (
17 operatorPrefix = "etcd_operator"
18 reconcileTotal = "reconcile_total"
19 reconcileErrorsTotal = "reconcile_errors_total"
20 etcdMemberCreateTotal = "etcdmember_create_total"
21 etcdMemberDeleteTotal = "etcdmember_delete_total"
22 etcdMemberReconcileConditionStatus = "etcdmember_reconcile_condition_status"
23 )
24
25 type Metrics struct {
26 Default metrics.Metrics
27 *Custom
28 }
29
30 type Custom struct {
31 counters map[string]*prometheus.CounterVec
32 reconcileConditionGauge *prometheus.GaugeVec
33 controllerPrefix string
34
35 incrementRequests chan recordFunc
36 }
37
38 type recordFunc func()
39
40 func New(mgr ctrl.Manager, prefix string) *Metrics {
41 custom := newCustom(prefix)
42
43 prefixes := []string{operatorPrefix, prefix}
44 fullPrefix := strings.Join(prefixes, "_")
45 def := metrics.New(mgr, fullPrefix,
46 metrics.WithSuspend(),
47 metrics.WithCollectors(
48 custom.Collectors()...,
49 ))
50
51 return &Metrics{
52 Default: def,
53 Custom: custom,
54 }
55 }
56
57 func newCustom(prefix string) *Custom {
58 prefixes := []string{operatorPrefix, prefix}
59 fullPrefix := strings.Join(prefixes, "_")
60 custom := &Custom{
61 counters: map[string]*prometheus.CounterVec{},
62 reconcileConditionGauge: newReconcileConditionGauge(fullPrefix),
63 controllerPrefix: prefix,
64 incrementRequests: make(chan recordFunc, 30),
65 }
66 custom.counters[reconcileTotal] = newReconcileTotal(fullPrefix)
67 custom.counters[reconcileErrorsTotal] = newReconcileErrorsTotal(fullPrefix)
68
69 if custom.controllerPrefix == "lifecycle" {
70 custom.counters[etcdMemberCreateTotal] = newEtcdMemberCreateTotal(fullPrefix)
71 custom.counters[etcdMemberDeleteTotal] = newEtcdMemberDeleteTotal(fullPrefix)
72 }
73
74 return custom
75 }
76
77 func newReconcileTotal(prefix string) *prometheus.CounterVec {
78 return prometheus.NewCounterVec(
79 prometheus.CounterOpts{
80 Name: metrics.Name(prefix, reconcileTotal),
81 Help: "Total number of reconciliation attempts for the etcd operator reconcilers",
82 },
83 []string{"reconciler", "resource"})
84 }
85
86 func newReconcileErrorsTotal(prefix string) *prometheus.CounterVec {
87 return prometheus.NewCounterVec(
88 prometheus.CounterOpts{
89 Name: metrics.Name(prefix, reconcileErrorsTotal),
90 Help: "Total number of reconciliation errors for the etcd operator reconcilers",
91 },
92 []string{"reconciler", "resource"})
93 }
94
95 func newEtcdMemberCreateTotal(prefix string) *prometheus.CounterVec {
96 return prometheus.NewCounterVec(
97 prometheus.CounterOpts{
98 Name: metrics.Name(prefix, etcdMemberCreateTotal),
99 Help: "Total number of times a new EtcdMember resource is created",
100 },
101 []string{"resource"})
102 }
103
104 func newEtcdMemberDeleteTotal(prefix string) *prometheus.CounterVec {
105 return prometheus.NewCounterVec(
106 prometheus.CounterOpts{
107 Name: metrics.Name(prefix, etcdMemberDeleteTotal),
108 Help: "Total number of times a new EtcdMember resource is deleted",
109 },
110 []string{"resource"})
111 }
112
113 func newReconcileConditionGauge(prefix string) *prometheus.GaugeVec {
114 return prometheus.NewGaugeVec(
115 prometheus.GaugeOpts{
116 Name: metrics.Name(prefix, etcdMemberReconcileConditionStatus),
117 Help: "EtcdMember reconcile condition status",
118 },
119 []string{"resource", "type", "status"})
120 }
121
122 func (c *Custom) Collectors() []prometheus.Collector {
123 collectors := []prometheus.Collector{
124 c.reconcileConditionGauge,
125 }
126 for _, counter := range c.counters {
127 collectors = append(collectors, counter)
128 }
129 return collectors
130 }
131
132 func (c *Custom) Run(members *v1etcd.EtcdMemberList) {
133 c.initialize(members)
134 go func() {
135 time.Sleep(60 * time.Second)
136 for request := range c.incrementRequests {
137 request()
138 }
139 }()
140 }
141
142 func (c *Custom) initialize(members *v1etcd.EtcdMemberList) {
143 for _, member := range members.Items {
144 if c.controllerPrefix != "inform" {
145 c.counters[reconcileTotal].WithLabelValues(c.controllerPrefix, member.Name).Add(0)
146 c.counters[reconcileErrorsTotal].WithLabelValues(c.controllerPrefix, member.Name).Add(0)
147 }
148 if c.controllerPrefix == "lifecycle" {
149 c.counters[etcdMemberCreateTotal].WithLabelValues(member.Name).Add(0)
150 c.counters[etcdMemberDeleteTotal].WithLabelValues(member.Name).Add(0)
151 }
152 }
153 }
154
155 func (c *Custom) RecordReconciliation(member *v1etcd.EtcdMember) {
156 c.incrementRequests <- func() {
157 c.counters[reconcileTotal].WithLabelValues(c.controllerPrefix, member.Name).Inc()
158 }
159 }
160
161 func (c *Custom) RecordReconciliationError(err error, member *v1etcd.EtcdMember) {
162 if err != nil {
163 c.incrementRequests <- func() {
164 c.counters[reconcileErrorsTotal].WithLabelValues(c.controllerPrefix, member.Name).Inc()
165 }
166 }
167 }
168
169 func (c *Custom) RecordEtcdMemberCreate(member *v1etcd.EtcdMember) {
170 c.incrementRequests <- func() {
171 c.counters[etcdMemberCreateTotal].WithLabelValues(member.Name).Inc()
172 }
173 }
174
175 func (c *Custom) RecordEtcdMemberDelete(member *v1etcd.EtcdMember) {
176 c.incrementRequests <- func() {
177 c.counters[etcdMemberDeleteTotal].WithLabelValues(member.Name).Inc()
178 }
179 }
180
181 func (c *Custom) RecordCondition(obj client.Object, condition string, status metav1.ConditionStatus, deleted bool) {
182 labels := prometheus.Labels{
183 "resource": obj.GetName(),
184 "type": condition,
185 }
186
187
188 if deleted {
189 c.reconcileConditionGauge.DeletePartialMatch(labels)
190 return
191 }
192
193 statuses := []metav1.ConditionStatus{metav1.ConditionTrue, metav1.ConditionFalse, metav1.ConditionUnknown}
194 for _, s := range statuses {
195 labels["status"] = string(s)
196 if status == s || (s == metav1.ConditionUnknown && string(status) == "") {
197 c.reconcileConditionGauge.With(labels).Set(1)
198 continue
199 }
200 c.reconcileConditionGauge.With(labels).Set(0)
201 }
202 }
203
View as plain text