inhibit_test.go

Documentation: github.com/prometheus/alertmanager/inhibit

     1  // Copyright 2016 Prometheus Team
     2  // Licensed under the Apache License, Version 2.0 (the "License");
     3  // you may not use this file except in compliance with the License.
     4  // You may obtain a copy of the License at
     5  //
     6  // http://www.apache.org/licenses/LICENSE-2.0
     7  //
     8  // Unless required by applicable law or agreed to in writing, software
     9  // distributed under the License is distributed on an "AS IS" BASIS,
    10  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package inhibit
    15  
    16  import (
    17  	"testing"
    18  	"time"
    19  
    20  	"github.com/go-kit/log"
    21  	"github.com/prometheus/client_golang/prometheus"
    22  	"github.com/prometheus/common/model"
    23  
    24  	"github.com/prometheus/alertmanager/config"
    25  	"github.com/prometheus/alertmanager/pkg/labels"
    26  	"github.com/prometheus/alertmanager/provider"
    27  	"github.com/prometheus/alertmanager/store"
    28  	"github.com/prometheus/alertmanager/types"
    29  )
    30  
    31  var nopLogger = log.NewNopLogger()
    32  
    33  func TestInhibitRuleHasEqual(t *testing.T) {
    34  	t.Parallel()
    35  
    36  	now := time.Now()
    37  	cases := []struct {
    38  		initial map[model.Fingerprint]*types.Alert
    39  		equal   model.LabelNames
    40  		input   model.LabelSet
    41  		result  bool
    42  	}{
    43  		{
    44  			// No source alerts at all.
    45  			initial: map[model.Fingerprint]*types.Alert{},
    46  			input:   model.LabelSet{"a": "b"},
    47  			result:  false,
    48  		},
    49  		{
    50  			// No equal labels, any source alerts satisfies the requirement.
    51  			initial: map[model.Fingerprint]*types.Alert{1: {}},
    52  			input:   model.LabelSet{"a": "b"},
    53  			result:  true,
    54  		},
    55  		{
    56  			// Matching but already resolved.
    57  			initial: map[model.Fingerprint]*types.Alert{
    58  				1: {
    59  					Alert: model.Alert{
    60  						Labels:   model.LabelSet{"a": "b", "b": "f"},
    61  						StartsAt: now.Add(-time.Minute),
    62  						EndsAt:   now.Add(-time.Second),
    63  					},
    64  				},
    65  				2: {
    66  					Alert: model.Alert{
    67  						Labels:   model.LabelSet{"a": "b", "b": "c"},
    68  						StartsAt: now.Add(-time.Minute),
    69  						EndsAt:   now.Add(-time.Second),
    70  					},
    71  				},
    72  			},
    73  			equal:  model.LabelNames{"a", "b"},
    74  			input:  model.LabelSet{"a": "b", "b": "c"},
    75  			result: false,
    76  		},
    77  		{
    78  			// Matching and unresolved.
    79  			initial: map[model.Fingerprint]*types.Alert{
    80  				1: {
    81  					Alert: model.Alert{
    82  						Labels:   model.LabelSet{"a": "b", "c": "d"},
    83  						StartsAt: now.Add(-time.Minute),
    84  						EndsAt:   now.Add(-time.Second),
    85  					},
    86  				},
    87  				2: {
    88  					Alert: model.Alert{
    89  						Labels:   model.LabelSet{"a": "b", "c": "f"},
    90  						StartsAt: now.Add(-time.Minute),
    91  						EndsAt:   now.Add(time.Hour),
    92  					},
    93  				},
    94  			},
    95  			equal:  model.LabelNames{"a"},
    96  			input:  model.LabelSet{"a": "b"},
    97  			result: true,
    98  		},
    99  		{
   100  			// Equal label does not match.
   101  			initial: map[model.Fingerprint]*types.Alert{
   102  				1: {
   103  					Alert: model.Alert{
   104  						Labels:   model.LabelSet{"a": "c", "c": "d"},
   105  						StartsAt: now.Add(-time.Minute),
   106  						EndsAt:   now.Add(-time.Second),
   107  					},
   108  				},
   109  				2: {
   110  					Alert: model.Alert{
   111  						Labels:   model.LabelSet{"a": "c", "c": "f"},
   112  						StartsAt: now.Add(-time.Minute),
   113  						EndsAt:   now.Add(-time.Second),
   114  					},
   115  				},
   116  			},
   117  			equal:  model.LabelNames{"a"},
   118  			input:  model.LabelSet{"a": "b"},
   119  			result: false,
   120  		},
   121  	}
   122  
   123  	for _, c := range cases {
   124  		r := &InhibitRule{
   125  			Equal:  map[model.LabelName]struct{}{},
   126  			scache: store.NewAlerts(),
   127  		}
   128  		for _, ln := range c.equal {
   129  			r.Equal[ln] = struct{}{}
   130  		}
   131  		for _, v := range c.initial {
   132  			r.scache.Set(v)
   133  		}
   134  
   135  		if _, have := r.hasEqual(c.input, false); have != c.result {
   136  			t.Errorf("Unexpected result %t, expected %t", have, c.result)
   137  		}
   138  	}
   139  }
   140  
   141  func TestInhibitRuleMatches(t *testing.T) {
   142  	t.Parallel()
   143  
   144  	rule1 := config.InhibitRule{
   145  		SourceMatch: map[string]string{"s1": "1"},
   146  		TargetMatch: map[string]string{"t1": "1"},
   147  		Equal:       model.LabelNames{"e"},
   148  	}
   149  	rule2 := config.InhibitRule{
   150  		SourceMatch: map[string]string{"s2": "1"},
   151  		TargetMatch: map[string]string{"t2": "1"},
   152  		Equal:       model.LabelNames{"e"},
   153  	}
   154  
   155  	m := types.NewMarker(prometheus.NewRegistry())
   156  	ih := NewInhibitor(nil, []*config.InhibitRule{&rule1, &rule2}, m, nopLogger)
   157  	now := time.Now()
   158  	// Active alert that matches the source filter of rule1.
   159  	sourceAlert1 := &types.Alert{
   160  		Alert: model.Alert{
   161  			Labels:   model.LabelSet{"s1": "1", "t1": "2", "e": "1"},
   162  			StartsAt: now.Add(-time.Minute),
   163  			EndsAt:   now.Add(time.Hour),
   164  		},
   165  	}
   166  	// Active alert that matches the source filter _and_ the target filter of rule2.
   167  	sourceAlert2 := &types.Alert{
   168  		Alert: model.Alert{
   169  			Labels:   model.LabelSet{"s2": "1", "t2": "1", "e": "1"},
   170  			StartsAt: now.Add(-time.Minute),
   171  			EndsAt:   now.Add(time.Hour),
   172  		},
   173  	}
   174  
   175  	ih.rules[0].scache = store.NewAlerts()
   176  	ih.rules[0].scache.Set(sourceAlert1)
   177  	ih.rules[1].scache = store.NewAlerts()
   178  	ih.rules[1].scache.Set(sourceAlert2)
   179  
   180  	cases := []struct {
   181  		target   model.LabelSet
   182  		expected bool
   183  	}{
   184  		{
   185  			// Matches target filter of rule1, inhibited.
   186  			target:   model.LabelSet{"t1": "1", "e": "1"},
   187  			expected: true,
   188  		},
   189  		{
   190  			// Matches target filter of rule2, inhibited.
   191  			target:   model.LabelSet{"t2": "1", "e": "1"},
   192  			expected: true,
   193  		},
   194  		{
   195  			// Matches target filter of rule1 (plus noise), inhibited.
   196  			target:   model.LabelSet{"t1": "1", "t3": "1", "e": "1"},
   197  			expected: true,
   198  		},
   199  		{
   200  			// Matches target filter of rule1 plus rule2, inhibited.
   201  			target:   model.LabelSet{"t1": "1", "t2": "1", "e": "1"},
   202  			expected: true,
   203  		},
   204  		{
   205  			// Doesn't match target filter, not inhibited.
   206  			target:   model.LabelSet{"t1": "0", "e": "1"},
   207  			expected: false,
   208  		},
   209  		{
   210  			// Matches both source and target filters of rule1,
   211  			// inhibited because sourceAlert1 matches only the
   212  			// source filter of rule1.
   213  			target:   model.LabelSet{"s1": "1", "t1": "1", "e": "1"},
   214  			expected: true,
   215  		},
   216  		{
   217  			// Matches both source and target filters of rule2,
   218  			// not inhibited because sourceAlert2 matches also both the
   219  			// source and target filter of rule2.
   220  			target:   model.LabelSet{"s2": "1", "t2": "1", "e": "1"},
   221  			expected: false,
   222  		},
   223  		{
   224  			// Matches target filter, equal label doesn't match, not inhibited
   225  			target:   model.LabelSet{"t1": "1", "e": "0"},
   226  			expected: false,
   227  		},
   228  	}
   229  
   230  	for _, c := range cases {
   231  		if actual := ih.Mutes(c.target); actual != c.expected {
   232  			t.Errorf("Expected (*Inhibitor).Mutes(%v) to return %t but got %t", c.target, c.expected, actual)
   233  		}
   234  	}
   235  }
   236  
   237  func TestInhibitRuleMatchers(t *testing.T) {
   238  	t.Parallel()
   239  
   240  	rule1 := config.InhibitRule{
   241  		SourceMatchers: config.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "s1", Value: "1"}},
   242  		TargetMatchers: config.Matchers{&labels.Matcher{Type: labels.MatchNotEqual, Name: "t1", Value: "1"}},
   243  		Equal:          model.LabelNames{"e"},
   244  	}
   245  	rule2 := config.InhibitRule{
   246  		SourceMatchers: config.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "s2", Value: "1"}},
   247  		TargetMatchers: config.Matchers{&labels.Matcher{Type: labels.MatchEqual, Name: "t2", Value: "1"}},
   248  		Equal:          model.LabelNames{"e"},
   249  	}
   250  
   251  	m := types.NewMarker(prometheus.NewRegistry())
   252  	ih := NewInhibitor(nil, []*config.InhibitRule{&rule1, &rule2}, m, nopLogger)
   253  	now := time.Now()
   254  	// Active alert that matches the source filter of rule1.
   255  	sourceAlert1 := &types.Alert{
   256  		Alert: model.Alert{
   257  			Labels:   model.LabelSet{"s1": "1", "t1": "2", "e": "1"},
   258  			StartsAt: now.Add(-time.Minute),
   259  			EndsAt:   now.Add(time.Hour),
   260  		},
   261  	}
   262  	// Active alert that matches the source filter _and_ the target filter of rule2.
   263  	sourceAlert2 := &types.Alert{
   264  		Alert: model.Alert{
   265  			Labels:   model.LabelSet{"s2": "1", "t2": "1", "e": "1"},
   266  			StartsAt: now.Add(-time.Minute),
   267  			EndsAt:   now.Add(time.Hour),
   268  		},
   269  	}
   270  
   271  	ih.rules[0].scache = store.NewAlerts()
   272  	ih.rules[0].scache.Set(sourceAlert1)
   273  	ih.rules[1].scache = store.NewAlerts()
   274  	ih.rules[1].scache.Set(sourceAlert2)
   275  
   276  	cases := []struct {
   277  		target   model.LabelSet
   278  		expected bool
   279  	}{
   280  		{
   281  			// Matches target filter of rule1, inhibited.
   282  			target:   model.LabelSet{"t1": "1", "e": "1"},
   283  			expected: false,
   284  		},
   285  		{
   286  			// Matches target filter of rule2, inhibited.
   287  			target:   model.LabelSet{"t2": "1", "e": "1"},
   288  			expected: true,
   289  		},
   290  		{
   291  			// Matches target filter of rule1 (plus noise), inhibited.
   292  			target:   model.LabelSet{"t1": "1", "t3": "1", "e": "1"},
   293  			expected: false,
   294  		},
   295  		{
   296  			// Matches target filter of rule1 plus rule2, inhibited.
   297  			target:   model.LabelSet{"t1": "1", "t2": "1", "e": "1"},
   298  			expected: true,
   299  		},
   300  		{
   301  			// Doesn't match target filter, not inhibited.
   302  			target:   model.LabelSet{"t1": "0", "e": "1"},
   303  			expected: true,
   304  		},
   305  		{
   306  			// Matches both source and target filters of rule1,
   307  			// inhibited because sourceAlert1 matches only the
   308  			// source filter of rule1.
   309  			target:   model.LabelSet{"s1": "1", "t1": "1", "e": "1"},
   310  			expected: false,
   311  		},
   312  		{
   313  			// Matches both source and target filters of rule2,
   314  			// not inhibited because sourceAlert2 matches also both the
   315  			// source and target filter of rule2.
   316  			target:   model.LabelSet{"s2": "1", "t2": "1", "e": "1"},
   317  			expected: true,
   318  		},
   319  		{
   320  			// Matches target filter, equal label doesn't match, not inhibited
   321  			target:   model.LabelSet{"t1": "1", "e": "0"},
   322  			expected: false,
   323  		},
   324  	}
   325  
   326  	for _, c := range cases {
   327  		if actual := ih.Mutes(c.target); actual != c.expected {
   328  			t.Errorf("Expected (*Inhibitor).Mutes(%v) to return %t but got %t", c.target, c.expected, actual)
   329  		}
   330  	}
   331  }
   332  
   333  type fakeAlerts struct {
   334  	alerts   []*types.Alert
   335  	finished chan struct{}
   336  }
   337  
   338  func newFakeAlerts(alerts []*types.Alert) *fakeAlerts {
   339  	return &fakeAlerts{
   340  		alerts:   alerts,
   341  		finished: make(chan struct{}),
   342  	}
   343  }
   344  
   345  func (f *fakeAlerts) GetPending() provider.AlertIterator          { return nil }
   346  func (f *fakeAlerts) Get(model.Fingerprint) (*types.Alert, error) { return nil, nil }
   347  func (f *fakeAlerts) Put(...*types.Alert) error                   { return nil }
   348  func (f *fakeAlerts) Subscribe() provider.AlertIterator {
   349  	ch := make(chan *types.Alert)
   350  	done := make(chan struct{})
   351  	go func() {
   352  		for _, a := range f.alerts {
   353  			ch <- a
   354  		}
   355  		// Send another (meaningless) alert to make sure that the inhibitor has
   356  		// processed everything.
   357  		ch <- &types.Alert{
   358  			Alert: model.Alert{
   359  				Labels:   model.LabelSet{},
   360  				StartsAt: time.Now(),
   361  			},
   362  		}
   363  		close(f.finished)
   364  		<-done
   365  	}()
   366  	return provider.NewAlertIterator(ch, done, nil)
   367  }
   368  
   369  func TestInhibit(t *testing.T) {
   370  	t.Parallel()
   371  
   372  	now := time.Now()
   373  	inhibitRule := func() *config.InhibitRule {
   374  		return &config.InhibitRule{
   375  			SourceMatch: map[string]string{"s": "1"},
   376  			TargetMatch: map[string]string{"t": "1"},
   377  			Equal:       model.LabelNames{"e"},
   378  		}
   379  	}
   380  	// alertOne is muted by alertTwo when it is active.
   381  	alertOne := func() *types.Alert {
   382  		return &types.Alert{
   383  			Alert: model.Alert{
   384  				Labels:   model.LabelSet{"t": "1", "e": "f"},
   385  				StartsAt: now.Add(-time.Minute),
   386  				EndsAt:   now.Add(time.Hour),
   387  			},
   388  		}
   389  	}
   390  	alertTwo := func(resolved bool) *types.Alert {
   391  		var end time.Time
   392  		if resolved {
   393  			end = now.Add(-time.Second)
   394  		} else {
   395  			end = now.Add(time.Hour)
   396  		}
   397  		return &types.Alert{
   398  			Alert: model.Alert{
   399  				Labels:   model.LabelSet{"s": "1", "e": "f"},
   400  				StartsAt: now.Add(-time.Minute),
   401  				EndsAt:   end,
   402  			},
   403  		}
   404  	}
   405  
   406  	type exp struct {
   407  		lbls  model.LabelSet
   408  		muted bool
   409  	}
   410  	for i, tc := range []struct {
   411  		alerts   []*types.Alert
   412  		expected []exp
   413  	}{
   414  		{
   415  			// alertOne shouldn't be muted since alertTwo hasn't fired.
   416  			alerts: []*types.Alert{alertOne()},
   417  			expected: []exp{
   418  				{
   419  					lbls:  model.LabelSet{"t": "1", "e": "f"},
   420  					muted: false,
   421  				},
   422  			},
   423  		},
   424  		{
   425  			// alertOne should be muted by alertTwo which is active.
   426  			alerts: []*types.Alert{alertOne(), alertTwo(false)},
   427  			expected: []exp{
   428  				{
   429  					lbls:  model.LabelSet{"t": "1", "e": "f"},
   430  					muted: true,
   431  				},
   432  				{
   433  					lbls:  model.LabelSet{"s": "1", "e": "f"},
   434  					muted: false,
   435  				},
   436  			},
   437  		},
   438  		{
   439  			// alertOne shouldn't be muted since alertTwo is resolved.
   440  			alerts: []*types.Alert{alertOne(), alertTwo(false), alertTwo(true)},
   441  			expected: []exp{
   442  				{
   443  					lbls:  model.LabelSet{"t": "1", "e": "f"},
   444  					muted: false,
   445  				},
   446  				{
   447  					lbls:  model.LabelSet{"s": "1", "e": "f"},
   448  					muted: false,
   449  				},
   450  			},
   451  		},
   452  	} {
   453  		ap := newFakeAlerts(tc.alerts)
   454  		mk := types.NewMarker(prometheus.NewRegistry())
   455  		inhibitor := NewInhibitor(ap, []*config.InhibitRule{inhibitRule()}, mk, nopLogger)
   456  
   457  		go func() {
   458  			for ap.finished != nil {
   459  				select {
   460  				case <-ap.finished:
   461  					ap.finished = nil
   462  				default:
   463  				}
   464  			}
   465  			inhibitor.Stop()
   466  		}()
   467  		inhibitor.Run()
   468  
   469  		for _, expected := range tc.expected {
   470  			if inhibitor.Mutes(expected.lbls) != expected.muted {
   471  				mute := "unmuted"
   472  				if expected.muted {
   473  					mute = "muted"
   474  				}
   475  				t.Errorf("tc: %d, expected alert with labels %q to be %s", i, expected.lbls, mute)
   476  			}
   477  		}
   478  	}
   479  }
   480
View as plain text