...

Source file src/edge-infra.dev/pkg/lib/gcp/monitoring/alerting/incident_test.go

Documentation: edge-infra.dev/pkg/lib/gcp/monitoring/alerting

     1  package alerting
     2  
     3  import (
     4  	"strings"
     5  	"testing"
     6  
     7  	"github.com/stretchr/testify/assert"
     8  )
     9  
    10  var (
    11  	metricExample = `{
    12  	"version":"test",
    13  	"incident":{
    14  		"incident_id":"12345",
    15  		"scoping_project_id":"12345",
    16  		"scoping_project_number":12345,
    17  		"url":"http://www.example.com",
    18  		"started_at":0,
    19  		"ended_at":0,
    20  		"state":"OPEN",
    21  		"summary":"Test Incident",
    22  		"apigee_url":"http://www.example.com",
    23  		"observed_value":"1.0",
    24  		"resource":{
    25  			"type":"example_resource",
    26  			"labels":{"example":"label"}
    27  		},
    28  		"resource_type_display_name":"Example Resource Type",
    29  		"resource_id":"12345",
    30  		"resource_display_name":"Example Resource",
    31  		"resource_name":"projects/12345/example_resources/12345",
    32  		"metric":{
    33  			"type":"test.googleapis.com/metric",
    34  			"displayName":"Test Metric",
    35  			"labels":{"example":"label"}
    36  		},
    37  		"metadata":{
    38  			"system_labels":{"example":"label"},
    39  			"user_labels":{"example":"label"}
    40  		},
    41  		"policy_name":"projects/12345/alertPolicies/12345",
    42  		"policy_user_labels":{"example":"label"},
    43  		"documentation":"Test documentation",
    44  		"condition":{
    45  			"name":"projects/12345/alertPolicies/12345/conditions/12345",
    46  			"displayName":"Example condition",
    47  			"conditionThreshold":{
    48  				"filter":"metric.type=\"test.googleapis.com/metric\" resource.type=\"example_resource\"",
    49  				"comparison":"COMPARISON_GT",
    50  				"thresholdValue":0.5,
    51  				"duration":"0s",
    52  				"trigger":{"count":1}
    53  			}
    54  		},
    55  		"condition_name":"Example condition",
    56  		"threshold_value":"0.5"}
    57  	}`
    58  
    59  	metricIncidentOpen = `{
    60  		"incident": {
    61  		  "condition": {
    62  			"conditionMonitoringQueryLanguage": {
    63  			  "duration": "300s",
    64  			  "query": "fetch prometheus_target\n| metric 'prometheus.googleapis.com/gotk_reconcile_condition/gauge'\n| filter\n    resource.project_id = 'ret-edge-vfdqa1gj666jpo8b61oa9'\n    &&\n    (metric.status == 'False' && metric.status != 'Deleted'\n    && metric.type == 'Ready' && metric.kind!~'HelmChart')\n| group_by 1m,\n    [value_gotk_reconcile_condition_max: max(value.gotk_reconcile_condition)]\n| every 1m\n| filter val() != 0\n| group_by\n    [metric.exported_namespace, metric.name, metric.kind, resource.project_id,\n    resource.cluster],\n    [value_gotk_reconcile_condition_max_max:\n      max(value_gotk_reconcile_condition_max)]\n| condition val() == 1"
    65  			},
    66  			"displayName": "PFJ Flux Reconcile Failures",
    67  			"name": "projects/ret-edge-stage1-foreman/alertPolicies/15678640162913868409/conditions/15678640162913867352"
    68  		  },
    69  		  "condition_name": "PFJ Flux Reconcile Failures",
    70  		  "documentation": {
    71  			"content": "Debugging steps can be found [here](https://docs.edge-infra.dev/runbooks/customer-alerts/alerts/#flux-resource-failures)",
    72  			"mime_type": "text/markdown"
    73  		  },
    74  		  "ended_at": null,
    75  		  "incident_id": "0.mk9kdylxogso",
    76  		  "metadata": {
    77  			"system_labels": {},
    78  			"user_labels": {}
    79  		  },
    80  		  "metric": {
    81  			"displayName": "",
    82  			"labels": {
    83  			  "exported_namespace": "flux-system",
    84  			  "kind": "HelmRelease",
    85  			  "name": "store"
    86  			},
    87  			"type": ""
    88  		  },
    89  		  "policy_name": "PFJ Flux Resource Failures",
    90  		  "policy_user_labels": {
    91  			"managed-by-cnrm": "true"
    92  		  },
    93  		  "resource": {
    94  			"labels": {
    95  			  "cluster": "s09570-ptc-qa-lab_v2",
    96  			  "project_id": "ret-edge-vfdqa1gj666jpo8b61oa9"
    97  			},
    98  			"type": "timeseries_query"
    99  		  },
   100  		  "resource_id": "",
   101  		  "resource_name": "ret-edge-vfdqa1gj666jpo8b61oa9 labels {project_id=ret-edge-vfdqa1gj666jpo8b61oa9, cluster=s09570-ptc-qa-lab_v2}",
   102  		  "resource_type_display_name": "",
   103  		  "scoping_project_id": "ret-edge-stage1-foreman",
   104  		  "scoping_project_number": 450891108899,
   105  		  "started_at": 1657649627,
   106  		  "state": "open",
   107  		  "summary": "PFJ Flux Resource Failures PFJ Flux Reconcile Failures alert for ret-edge-vfdqa1gj666jpo8b61oa9 labels {project_id=ret-edge-vfdqa1gj666jpo8b61oa9, cluster=s09570-ptc-qa-lab_v2} with metric labels {exported_namespace=flux-system, kind=HelmRelease, name=store} started.",
   108  		  "url": "https://console.cloud.google.com/monitoring/alerting/incidents/0.mk9kdylxogso?project=ret-edge-stage1-foreman"
   109  		},
   110  		"version": "1.2"
   111  	  }`
   112  
   113  	metricIncidentClosed = `{
   114  		"incident": {
   115  		  "condition": {
   116  			"conditionMonitoringQueryLanguage": {
   117  			  "duration": "300s",
   118  			  "query": "fetch prometheus_target\n| metric 'prometheus.googleapis.com/gotk_reconcile_condition/gauge'\n| filter\n    resource.project_id =~ 'ret-edge-g0k2qwvckmu1j1vkhg0hl|ret-edge-gbwujre2kp8rhmd1rw70l'\n    &&\n    (metric.status == 'False' && metric.status != 'Deleted'\n    && metric.type == 'Ready' && metric.kind!~'HelmChart')\n| group_by 1m,\n    [value_gotk_reconcile_condition_max: max(value.gotk_reconcile_condition)]\n| every 1m\n| filter val() != 0\n| group_by\n    [metric.exported_namespace, metric.name, metric.kind, resource.project_id,\n    resource.cluster],\n    [value_gotk_reconcile_condition_max_max:\n      max(value_gotk_reconcile_condition_max)]\n| condition val() == 1"
   119  			},
   120  			"displayName": "RBS Flux Reconcile Failures",
   121  			"name": "projects/ret-edge-stage1-foreman/alertPolicies/14331101202040885318/conditions/14331101202040888963"
   122  		  },
   123  		  "condition_name": "RBS Flux Reconcile Failures",
   124  		  "documentation": {
   125  			"content": "Debugging steps can be found [here](https://docs.edge-infra.dev/runbooks/customer-alerts/alerts/#flux-resource-failures)",
   126  			"mime_type": "text/markdown"
   127  		  },
   128  		  "ended_at": 1657650903,
   129  		  "incident_id": "0.mk9dmg7m8ohy",
   130  		  "metadata": {
   131  			"system_labels": {},
   132  			"user_labels": {}
   133  		  },
   134  		  "metric": {
   135  			"displayName": "",
   136  			"labels": {
   137  			  "exported_namespace": "edge-flux",
   138  			  "kind": "Bucket",
   139  			  "name": "xmv77jmg1erz-cluster-sync"
   140  			},
   141  			"type": ""
   142  		  },
   143  		  "policy_name": "RBS Flux Resource Failures",
   144  		  "policy_user_labels": {
   145  			"managed-by-cnrm": "true"
   146  		  },
   147  		  "resource": {
   148  			"labels": {
   149  			  "cluster": "Store 9915",
   150  			  "project_id": "ret-edge-gbwujre2kp8rhmd1rw70l"
   151  			},
   152  			"type": "timeseries_query"
   153  		  },
   154  		  "resource_id": "",
   155  		  "resource_name": "ret-edge-gbwujre2kp8rhmd1rw70l labels {project_id=ret-edge-gbwujre2kp8rhmd1rw70l, cluster=Store 9915}",
   156  		  "resource_type_display_name": "",
   157  		  "scoping_project_id": "ret-edge-stage1-foreman",
   158  		  "scoping_project_number": 611910449365,
   159  		  "started_at": 1657631855,
   160  		  "state": "closed",
   161  		  "summary": "RBS Flux Resource Failures RBS Flux Reconcile Failures alert for ret-edge-gbwujre2kp8rhmd1rw70l labels {project_id=ret-edge-gbwujre2kp8rhmd1rw70l, cluster=Store 9915} with metric labels {exported_namespace=edge-flux, kind=Bucket, name=xmv77jmg1erz-cluster-sync} resolved.",
   162  		  "url": "https://console.cloud.google.com/monitoring/alerting/incidents/0.mk9dmg7m8ohy?project=ret-edge-stage1-foreman"
   163  		},
   164  		"version": "1.2"
   165  	  }`
   166  )
   167  
   168  func TestUnmarshal(t *testing.T) {
   169  	testCases := []struct {
   170  		description string
   171  		incident    string
   172  		expectError bool
   173  	}{ // todo - add more cases
   174  		{
   175  			"open metric incident",
   176  			metricIncidentOpen,
   177  			false,
   178  		}, {
   179  			"closed metric incident",
   180  			metricIncidentClosed,
   181  			false,
   182  		}, {
   183  			"test incident, sent when testing connection",
   184  			metricExample,
   185  			true,
   186  		},
   187  	}
   188  
   189  	for _, tc := range testCases {
   190  		// remove whitespace so it's easier to compare results
   191  		incTrimmed := strings.ReplaceAll(tc.incident, "\n", "")
   192  		incTrimmed = strings.ReplaceAll(incTrimmed, "\t", "")
   193  		incTrimmed = strings.ReplaceAll(incTrimmed, " ", "")
   194  		data := []byte(incTrimmed)
   195  
   196  		i, err := Unmarshal(data)
   197  		if tc.expectError {
   198  			assert.Error(t, err, tc.description)
   199  		} else {
   200  			assert.NoError(t, err, tc.description)
   201  
   202  			actualData, err := Marshall(i)
   203  			assert.NoError(t, err)
   204  
   205  			expectedComparibleForm := strings.TrimSpace(string(data))
   206  			actualComparibleForm := strings.TrimSpace(string(actualData))
   207  			assert.Equal(t, expectedComparibleForm, actualComparibleForm)
   208  		}
   209  	}
   210  }
   211  

View as plain text