package metrics import ( "strings" "time" "github.com/prometheus/client_golang/prometheus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "edge-infra.dev/pkg/k8s/runtime/controller/metrics" v1etcd "edge-infra.dev/pkg/sds/etcd/operator/apis/etcdmember/v1" ) const ( operatorPrefix = "etcd_operator" reconcileTotal = "reconcile_total" reconcileErrorsTotal = "reconcile_errors_total" etcdMemberCreateTotal = "etcdmember_create_total" etcdMemberDeleteTotal = "etcdmember_delete_total" etcdMemberReconcileConditionStatus = "etcdmember_reconcile_condition_status" ) type Metrics struct { Default metrics.Metrics *Custom } type Custom struct { counters map[string]*prometheus.CounterVec reconcileConditionGauge *prometheus.GaugeVec controllerPrefix string incrementRequests chan recordFunc } type recordFunc func() func New(mgr ctrl.Manager, prefix string) *Metrics { custom := newCustom(prefix) prefixes := []string{operatorPrefix, prefix} fullPrefix := strings.Join(prefixes, "_") def := metrics.New(mgr, fullPrefix, metrics.WithSuspend(), metrics.WithCollectors( custom.Collectors()..., )) return &Metrics{ Default: def, Custom: custom, } } func newCustom(prefix string) *Custom { prefixes := []string{operatorPrefix, prefix} fullPrefix := strings.Join(prefixes, "_") custom := &Custom{ counters: map[string]*prometheus.CounterVec{}, reconcileConditionGauge: newReconcileConditionGauge(fullPrefix), controllerPrefix: prefix, incrementRequests: make(chan recordFunc, 30), } custom.counters[reconcileTotal] = newReconcileTotal(fullPrefix) custom.counters[reconcileErrorsTotal] = newReconcileErrorsTotal(fullPrefix) if custom.controllerPrefix == "lifecycle" { custom.counters[etcdMemberCreateTotal] = newEtcdMemberCreateTotal(fullPrefix) custom.counters[etcdMemberDeleteTotal] = newEtcdMemberDeleteTotal(fullPrefix) } return custom } func newReconcileTotal(prefix string) *prometheus.CounterVec { return prometheus.NewCounterVec( prometheus.CounterOpts{ Name: metrics.Name(prefix, reconcileTotal), Help: "Total number of reconciliation attempts for the etcd operator reconcilers", }, []string{"reconciler", "resource"}) } func newReconcileErrorsTotal(prefix string) *prometheus.CounterVec { return prometheus.NewCounterVec( prometheus.CounterOpts{ Name: metrics.Name(prefix, reconcileErrorsTotal), Help: "Total number of reconciliation errors for the etcd operator reconcilers", }, []string{"reconciler", "resource"}) } func newEtcdMemberCreateTotal(prefix string) *prometheus.CounterVec { return prometheus.NewCounterVec( prometheus.CounterOpts{ Name: metrics.Name(prefix, etcdMemberCreateTotal), Help: "Total number of times a new EtcdMember resource is created", }, []string{"resource"}) } func newEtcdMemberDeleteTotal(prefix string) *prometheus.CounterVec { return prometheus.NewCounterVec( prometheus.CounterOpts{ Name: metrics.Name(prefix, etcdMemberDeleteTotal), Help: "Total number of times a new EtcdMember resource is deleted", }, []string{"resource"}) } func newReconcileConditionGauge(prefix string) *prometheus.GaugeVec { return prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: metrics.Name(prefix, etcdMemberReconcileConditionStatus), Help: "EtcdMember reconcile condition status", }, []string{"resource", "type", "status"}) } func (c *Custom) Collectors() []prometheus.Collector { collectors := []prometheus.Collector{ c.reconcileConditionGauge, } for _, counter := range c.counters { collectors = append(collectors, counter) } return collectors } func (c *Custom) Run(members *v1etcd.EtcdMemberList) { c.initialize(members) go func() { time.Sleep(60 * time.Second) for request := range c.incrementRequests { request() } }() } func (c *Custom) initialize(members *v1etcd.EtcdMemberList) { for _, member := range members.Items { if c.controllerPrefix != "inform" { c.counters[reconcileTotal].WithLabelValues(c.controllerPrefix, member.Name).Add(0) c.counters[reconcileErrorsTotal].WithLabelValues(c.controllerPrefix, member.Name).Add(0) } if c.controllerPrefix == "lifecycle" { c.counters[etcdMemberCreateTotal].WithLabelValues(member.Name).Add(0) c.counters[etcdMemberDeleteTotal].WithLabelValues(member.Name).Add(0) } } } func (c *Custom) RecordReconciliation(member *v1etcd.EtcdMember) { c.incrementRequests <- func() { c.counters[reconcileTotal].WithLabelValues(c.controllerPrefix, member.Name).Inc() } } func (c *Custom) RecordReconciliationError(err error, member *v1etcd.EtcdMember) { if err != nil { c.incrementRequests <- func() { c.counters[reconcileErrorsTotal].WithLabelValues(c.controllerPrefix, member.Name).Inc() } } } func (c *Custom) RecordEtcdMemberCreate(member *v1etcd.EtcdMember) { c.incrementRequests <- func() { c.counters[etcdMemberCreateTotal].WithLabelValues(member.Name).Inc() } } func (c *Custom) RecordEtcdMemberDelete(member *v1etcd.EtcdMember) { c.incrementRequests <- func() { c.counters[etcdMemberDeleteTotal].WithLabelValues(member.Name).Inc() } } func (c *Custom) RecordCondition(obj client.Object, condition string, status metav1.ConditionStatus, deleted bool) { labels := prometheus.Labels{ "resource": obj.GetName(), "type": condition, } // If the object is deleted, remove all corresponding metrics if deleted { c.reconcileConditionGauge.DeletePartialMatch(labels) return } statuses := []metav1.ConditionStatus{metav1.ConditionTrue, metav1.ConditionFalse, metav1.ConditionUnknown} for _, s := range statuses { labels["status"] = string(s) if status == s || (s == metav1.ConditionUnknown && string(status) == "") { c.reconcileConditionGauge.With(labels).Set(1) continue } c.reconcileConditionGauge.With(labels).Set(0) } }