...

Source file src/edge-infra.dev/pkg/sds/etcd/operator/internal/metrics/metrics.go

Documentation: edge-infra.dev/pkg/sds/etcd/operator/internal/metrics

     1  package metrics
     2  
     3  import (
     4  	"strings"
     5  	"time"
     6  
     7  	"github.com/prometheus/client_golang/prometheus"
     8  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
     9  	ctrl "sigs.k8s.io/controller-runtime"
    10  	"sigs.k8s.io/controller-runtime/pkg/client"
    11  
    12  	"edge-infra.dev/pkg/k8s/runtime/controller/metrics"
    13  	v1etcd "edge-infra.dev/pkg/sds/etcd/operator/apis/etcdmember/v1"
    14  )
    15  
    16  const (
    17  	operatorPrefix                     = "etcd_operator"
    18  	reconcileTotal                     = "reconcile_total"
    19  	reconcileErrorsTotal               = "reconcile_errors_total"
    20  	etcdMemberCreateTotal              = "etcdmember_create_total"
    21  	etcdMemberDeleteTotal              = "etcdmember_delete_total"
    22  	etcdMemberReconcileConditionStatus = "etcdmember_reconcile_condition_status"
    23  )
    24  
    25  type Metrics struct {
    26  	Default metrics.Metrics
    27  	*Custom
    28  }
    29  
    30  type Custom struct {
    31  	counters                map[string]*prometheus.CounterVec
    32  	reconcileConditionGauge *prometheus.GaugeVec
    33  	controllerPrefix        string
    34  
    35  	incrementRequests chan recordFunc
    36  }
    37  
    38  type recordFunc func()
    39  
    40  func New(mgr ctrl.Manager, prefix string) *Metrics {
    41  	custom := newCustom(prefix)
    42  
    43  	prefixes := []string{operatorPrefix, prefix}
    44  	fullPrefix := strings.Join(prefixes, "_")
    45  	def := metrics.New(mgr, fullPrefix,
    46  		metrics.WithSuspend(),
    47  		metrics.WithCollectors(
    48  			custom.Collectors()...,
    49  		))
    50  
    51  	return &Metrics{
    52  		Default: def,
    53  		Custom:  custom,
    54  	}
    55  }
    56  
    57  func newCustom(prefix string) *Custom {
    58  	prefixes := []string{operatorPrefix, prefix}
    59  	fullPrefix := strings.Join(prefixes, "_")
    60  	custom := &Custom{
    61  		counters:                map[string]*prometheus.CounterVec{},
    62  		reconcileConditionGauge: newReconcileConditionGauge(fullPrefix),
    63  		controllerPrefix:        prefix,
    64  		incrementRequests:       make(chan recordFunc, 30),
    65  	}
    66  	custom.counters[reconcileTotal] = newReconcileTotal(fullPrefix)
    67  	custom.counters[reconcileErrorsTotal] = newReconcileErrorsTotal(fullPrefix)
    68  
    69  	if custom.controllerPrefix == "lifecycle" {
    70  		custom.counters[etcdMemberCreateTotal] = newEtcdMemberCreateTotal(fullPrefix)
    71  		custom.counters[etcdMemberDeleteTotal] = newEtcdMemberDeleteTotal(fullPrefix)
    72  	}
    73  
    74  	return custom
    75  }
    76  
    77  func newReconcileTotal(prefix string) *prometheus.CounterVec {
    78  	return prometheus.NewCounterVec(
    79  		prometheus.CounterOpts{
    80  			Name: metrics.Name(prefix, reconcileTotal),
    81  			Help: "Total number of reconciliation attempts for the etcd operator reconcilers",
    82  		},
    83  		[]string{"reconciler", "resource"})
    84  }
    85  
    86  func newReconcileErrorsTotal(prefix string) *prometheus.CounterVec {
    87  	return prometheus.NewCounterVec(
    88  		prometheus.CounterOpts{
    89  			Name: metrics.Name(prefix, reconcileErrorsTotal),
    90  			Help: "Total number of reconciliation errors for the etcd operator reconcilers",
    91  		},
    92  		[]string{"reconciler", "resource"})
    93  }
    94  
    95  func newEtcdMemberCreateTotal(prefix string) *prometheus.CounterVec {
    96  	return prometheus.NewCounterVec(
    97  		prometheus.CounterOpts{
    98  			Name: metrics.Name(prefix, etcdMemberCreateTotal),
    99  			Help: "Total number of times a new EtcdMember resource is created",
   100  		},
   101  		[]string{"resource"})
   102  }
   103  
   104  func newEtcdMemberDeleteTotal(prefix string) *prometheus.CounterVec {
   105  	return prometheus.NewCounterVec(
   106  		prometheus.CounterOpts{
   107  			Name: metrics.Name(prefix, etcdMemberDeleteTotal),
   108  			Help: "Total number of times a new EtcdMember resource is deleted",
   109  		},
   110  		[]string{"resource"})
   111  }
   112  
   113  func newReconcileConditionGauge(prefix string) *prometheus.GaugeVec {
   114  	return prometheus.NewGaugeVec(
   115  		prometheus.GaugeOpts{
   116  			Name: metrics.Name(prefix, etcdMemberReconcileConditionStatus),
   117  			Help: "EtcdMember reconcile condition status",
   118  		},
   119  		[]string{"resource", "type", "status"})
   120  }
   121  
   122  func (c *Custom) Collectors() []prometheus.Collector {
   123  	collectors := []prometheus.Collector{
   124  		c.reconcileConditionGauge,
   125  	}
   126  	for _, counter := range c.counters {
   127  		collectors = append(collectors, counter)
   128  	}
   129  	return collectors
   130  }
   131  
   132  func (c *Custom) Run(members *v1etcd.EtcdMemberList) {
   133  	c.initialize(members)
   134  	go func() {
   135  		time.Sleep(60 * time.Second)
   136  		for request := range c.incrementRequests {
   137  			request()
   138  		}
   139  	}()
   140  }
   141  
   142  func (c *Custom) initialize(members *v1etcd.EtcdMemberList) {
   143  	for _, member := range members.Items {
   144  		if c.controllerPrefix != "inform" {
   145  			c.counters[reconcileTotal].WithLabelValues(c.controllerPrefix, member.Name).Add(0)
   146  			c.counters[reconcileErrorsTotal].WithLabelValues(c.controllerPrefix, member.Name).Add(0)
   147  		}
   148  		if c.controllerPrefix == "lifecycle" {
   149  			c.counters[etcdMemberCreateTotal].WithLabelValues(member.Name).Add(0)
   150  			c.counters[etcdMemberDeleteTotal].WithLabelValues(member.Name).Add(0)
   151  		}
   152  	}
   153  }
   154  
   155  func (c *Custom) RecordReconciliation(member *v1etcd.EtcdMember) {
   156  	c.incrementRequests <- func() {
   157  		c.counters[reconcileTotal].WithLabelValues(c.controllerPrefix, member.Name).Inc()
   158  	}
   159  }
   160  
   161  func (c *Custom) RecordReconciliationError(err error, member *v1etcd.EtcdMember) {
   162  	if err != nil {
   163  		c.incrementRequests <- func() {
   164  			c.counters[reconcileErrorsTotal].WithLabelValues(c.controllerPrefix, member.Name).Inc()
   165  		}
   166  	}
   167  }
   168  
   169  func (c *Custom) RecordEtcdMemberCreate(member *v1etcd.EtcdMember) {
   170  	c.incrementRequests <- func() {
   171  		c.counters[etcdMemberCreateTotal].WithLabelValues(member.Name).Inc()
   172  	}
   173  }
   174  
   175  func (c *Custom) RecordEtcdMemberDelete(member *v1etcd.EtcdMember) {
   176  	c.incrementRequests <- func() {
   177  		c.counters[etcdMemberDeleteTotal].WithLabelValues(member.Name).Inc()
   178  	}
   179  }
   180  
   181  func (c *Custom) RecordCondition(obj client.Object, condition string, status metav1.ConditionStatus, deleted bool) {
   182  	labels := prometheus.Labels{
   183  		"resource": obj.GetName(),
   184  		"type":     condition,
   185  	}
   186  
   187  	// If the object is deleted, remove all corresponding metrics
   188  	if deleted {
   189  		c.reconcileConditionGauge.DeletePartialMatch(labels)
   190  		return
   191  	}
   192  
   193  	statuses := []metav1.ConditionStatus{metav1.ConditionTrue, metav1.ConditionFalse, metav1.ConditionUnknown}
   194  	for _, s := range statuses {
   195  		labels["status"] = string(s)
   196  		if status == s || (s == metav1.ConditionUnknown && string(status) == "") {
   197  			c.reconcileConditionGauge.With(labels).Set(1)
   198  			continue
   199  		}
   200  		c.reconcileConditionGauge.With(labels).Set(0)
   201  	}
   202  }
   203  

View as plain text