am-webhook.go

Documentation: edge-infra.dev/pkg/edge/monitoring/am-webhook

     1  package amwebhook
     2  
     3  import (
     4  	"context"
     5  	"encoding/json"
     6  	"net/http"
     7  	"strings"
     8  
     9  	"github.com/go-logr/logr"
    10  	"github.com/prometheus/alertmanager/template"
    11  
    12  	"edge-infra.dev/pkg/lib/logging"
    13  
    14  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    15  	controllerruntime "sigs.k8s.io/controller-runtime"
    16  	"sigs.k8s.io/controller-runtime/pkg/client"
    17  
    18  	corev1 "k8s.io/api/core/v1"
    19  )
    20  
    21  // +kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get;patch
    22  // +kubebuilder:rbac:groups="",resources=pods,verbs=get;delete
    23  
    24  func AlertWebhook(_ http.ResponseWriter, req *http.Request) {
    25  	log := logging.NewLogger().WithName("am-webhook")
    26  	defer req.Body.Close()
    27  
    28  	data := template.Data{}
    29  	if err := json.NewDecoder(req.Body).Decode(&data); err != nil {
    30  		log.Error(err, "Alerts: GroupLabels", data.GroupLabels, "CommonLabels", data.CommonLabels)
    31  	}
    32  
    33  	for _, alert := range data.Alerts {
    34  		severity := strings.ToUpper(alert.Labels["severity"])
    35  		switch severity {
    36  		case "CRITICAL":
    37  			if _, ok := alert.Labels["action"]; ok {
    38  				if err := action(req.Context(), alert, &log); err != nil {
    39  					log.Error(err, "action executed unsuccessfully")
    40  				} else {
    41  					log.Info("action executed successfully")
    42  				}
    43  			}
    44  		default:
    45  			log.Info("no action on app")
    46  		}
    47  	}
    48  }
    49  
    50  func action(ctx context.Context, alert template.Alert, log *logr.Logger) error {
    51  	// Getting the config to use with the client
    52  	config, err := controllerruntime.GetConfig()
    53  	if err != nil {
    54  		log.Error(err, "could not get kube config")
    55  		return err
    56  	}
    57  
    58  	// Creating the client
    59  	alertClient, err := client.New(config, client.Options{})
    60  	if err != nil {
    61  		log.Error(err, "could not create the client")
    62  		return err
    63  	}
    64  
    65  	action := alert.Labels["action"]
    66  
    67  	switch action {
    68  	case "restart-pod":
    69  		pod := &corev1.Pod{
    70  			ObjectMeta: metav1.ObjectMeta{
    71  				Name:      alert.Labels["pod"],
    72  				Namespace: alert.Labels["namespace"],
    73  			},
    74  		}
    75  
    76  		if err := alertClient.Delete(ctx, pod); err != nil {
    77  			log.Error(err, "failed to delete pod")
    78  			return err
    79  		}
    80  	default:
    81  		log.Info("no action to perform")
    82  	}
    83  
    84  	return nil
    85  }
    86
View as plain text