package metrics import ( "fmt" "time" "github.com/prometheus/client_golang/prometheus" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( reconcileDurationBucketsMin = 0.05 reconcileDurationBucketsMax = 600.0 reconcileDurationBucketsCount = 10 ) // Name returns a standard Edge controller metric name using the component name // and base metric name. It prepends a standard edge-wide metric prefix. // // Note: If https://github.com/kubernetes-sigs/controller-runtime/issues/1995 // is resolved, this function can be deprecated in favor of using // prometheus.WrapRegistererWithPrefix(). func Name(prefix, metricName string) string { return fmt.Sprintf("edge_%s_%s", prefix, metricName) } // Recorder is a standard metrics collector for K8s controllers. It contains // pre-defined collectors that are instantiated via NewRecorder() by default. // In general, the expectation is that the Recorder struct would only be used by // controller authors via the Metrics struct. type Recorder struct { reconcileConditionGauge *prometheus.GaugeVec reconcileConditionGaugeWithReason *prometheus.GaugeVec suspendGauge *prometheus.GaugeVec durationHistogram *prometheus.HistogramVec // additional user-provided collectors, for specific controllers collectors []prometheus.Collector } // Create a new Recorder for recording standard metrics and any additional metrics // added via WithCollectors(). The metrics prefix string is appended after a // standard edge prefix. e.g., a prefix of "cluster" would produce metric names // like "edge_cluster_reconcile_condition_status", etc. Because any additional // prometheus.Collectors don't make their metric name visible via the Collector // interface, we cannot add the same prefix for additional custom collectors // (see https://github.com/kubernetes-sigs/controller-runtime/issues/1995). // For custom collectors, its recommended to use the Name() function to create // consistent metric names. func NewRecorder(prefix string, options ...Option) *Recorder { opts := makeOptions(options...) rec := &Recorder{collectors: opts.customCollectors} if opts.reason { // If WithReason() is passed during setup, reconcile_condition_status will also include the `reason` label. rec.reconcileConditionGaugeWithReason = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: Name(prefix, "reconcile_condition_status"), Help: "The current condition status of a controller's resource reconciliation with reason included.", }, []string{"kind", "name", "namespace", "type", "reason", "status"}, ) rec.collectors = append(rec.collectors, rec.reconcileConditionGaugeWithReason) } else { rec.reconcileConditionGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: Name(prefix, "reconcile_condition_status"), Help: "The current condition status of a controller's resource reconciliation.", }, []string{"kind", "name", "namespace", "type", "status"}, ) rec.collectors = append(rec.collectors, rec.reconcileConditionGauge) } // set histogram buckets to have more useful sizes rec.durationHistogram = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: Name(prefix, "reconcile_duration_seconds"), Help: "The duration in seconds of a controller's resource reconciliation.", Buckets: prometheus.ExponentialBucketsRange(reconcileDurationBucketsMin, reconcileDurationBucketsMax, reconcileDurationBucketsCount), }, []string{"kind", "name", "namespace"}, ) rec.collectors = append(rec.collectors, rec.durationHistogram) // add optional recorders if opts.suspend { rec.suspendGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: Name(prefix, "reconcile_suspend_status"), Help: "The current suspend status of a controller's resource.", }, []string{"kind", "name", "namespace"}, ) rec.collectors = append(rec.collectors, rec.suspendGauge) } return rec } // Collectors returns a slice of Prometheus collectors, which can be used to // register them in a metrics registry. // TODO: Create collectors iteratively for each desired non-standard metric func (r *Recorder) Collectors() []prometheus.Collector { return r.collectors } // RecordDuration records the duration since start for the given ref. func (r *Recorder) RecordDuration(ref corev1.ObjectReference, start time.Time) { r.durationHistogram.WithLabelValues(ref.Kind, ref.Name, ref.Namespace).Observe(time.Since(start).Seconds()) } // RecordSuspend records the suspend status as given for the ref. func (r *Recorder) RecordSuspend(ref corev1.ObjectReference, suspend bool) error { // If suspendGauge is not set for the recorder, return if r.suspendGauge == nil { return fmt.Errorf("suspendGauge not set") } var value float64 if suspend { value = 1 } r.suspendGauge.WithLabelValues(ref.Kind, ref.Name, ref.Namespace).Set(value) return nil } // RecordCondition records the condition as given for the ref. If the object is // marked for deletion, the metrics for the referenced object are deleted. func (r *Recorder) RecordCondition(ref corev1.ObjectReference, condition metav1.Condition, deleted bool) error { // If reconcileConditionGauge is not set for the recorder, return if r.reconcileConditionGauge == nil { return fmt.Errorf("reconcileConditionGauge not set") } labels := prometheus.Labels{ "kind": ref.Kind, "name": ref.Name, "namespace": ref.Namespace, "type": condition.Type, } for _, status := range []string{string(metav1.ConditionTrue), string(metav1.ConditionFalse), string(metav1.ConditionUnknown)} { var value float64 labels["status"] = status if deleted { r.reconcileConditionGauge.DeletePartialMatch(labels) } else { if status == string(condition.Status) { value = 1 } r.reconcileConditionGauge.With(labels).Set(value) } } return nil } // RecordConditionWithReason records the condition as given for the ref. // If the object is marked for deletion, the metrics for the referenced object are deleted. // TODO(dk185217): RecordCondition should be deprecated in favor of this approach func (r *Recorder) RecordConditionWithReason(ref corev1.ObjectReference, condition metav1.Condition, deleted bool) error { if deleted { return nil } labels := prometheus.Labels{ "kind": ref.Kind, "name": ref.Name, "namespace": ref.Namespace, "type": condition.Type, "status": string(condition.Status), "reason": condition.Reason, } r.reconcileConditionGaugeWithReason.With(labels).Set(1) return nil }