prometheus_stackdriver_controller_test.go

Documentation: edge-infra.dev/pkg/edge/monitoring/k8s/controllers/prometheusctl

     1  package prometheusctl
     2  
     3  import (
     4  	"context"
     5  	_ "embed"
     6  	"fmt"
     7  	"os"
     8  	"strings"
     9  	"testing"
    10  	"time"
    11  
    12  	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
    13  	"github.com/stretchr/testify/assert"
    14  	"github.com/stretchr/testify/suite"
    15  	corev1 "k8s.io/api/core/v1"
    16  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    17  	ctrl "sigs.k8s.io/controller-runtime"
    18  
    19  	"edge-infra.dev/pkg/edge/constants/api/cluster"
    20  	"edge-infra.dev/pkg/f8n/warehouse/pallet"
    21  	"edge-infra.dev/pkg/k8s/decoder"
    22  	"edge-infra.dev/pkg/k8s/meta"
    23  	eclient "edge-infra.dev/pkg/k8s/runtime/client"
    24  	"edge-infra.dev/pkg/k8s/runtime/controller"
    25  	"edge-infra.dev/pkg/k8s/unstructured"
    26  	"edge-infra.dev/pkg/lib/logging"
    27  	"edge-infra.dev/test"
    28  	"edge-infra.dev/test/framework"
    29  	"edge-infra.dev/test/framework/k8s"
    30  	"edge-infra.dev/test/framework/k8s/envtest"
    31  )
    32  
    33  var (
    34  	// shared test state
    35  	testEnv  *envtest.Environment
    36  	monitors []*unstructured.Unstructured
    37  
    38  	//go:embed testdata/expected-metrics.txt
    39  	expectedMetricsData string
    40  	//go:embed testdata/monitors.yaml
    41  	monitorsData []byte
    42  )
    43  
    44  type Suite struct {
    45  	*framework.Framework
    46  	*k8s.K8s
    47  
    48  	ctx     context.Context
    49  	timeout time.Duration
    50  	tick    time.Duration
    51  
    52  	// reconciler configuration
    53  	cfg config
    54  
    55  	expectedMetrics []string
    56  }
    57  
    58  func TestMain(m *testing.M) {
    59  	ctrl.SetLogger(logging.NewLogger().Logger)
    60  	// register framework flags + parse them, must be done before actual test
    61  	// execution
    62  	framework.HandleFlags()
    63  	// execute test suite
    64  	os.Exit(m.Run())
    65  }
    66  
    67  func TestController(t *testing.T) {
    68  	// initialize test environment
    69  	testEnv = envtest.Setup()
    70  
    71  	// parse embedded test assets
    72  	var err error
    73  	monitors, err = decoder.DecodeYAML(monitorsData)
    74  	test.NoError(err)
    75  
    76  	// run tests
    77  	t.Run(cluster.GKE, func(t *testing.T) {
    78  		s := buildSuite(cluster.GKE)
    79  		suite.Run(t, s)
    80  	})
    81  
    82  	t.Run(cluster.Generic, func(t *testing.T) {
    83  		s := buildSuite(cluster.Generic)
    84  		suite.Run(t, s)
    85  	})
    86  
    87  	t.Cleanup(func() {
    88  		test.NoError(testEnv.Stop())
    89  	})
    90  }
    91  
    92  func (s *Suite) TestPrometheusPatch() {
    93  	// TODO: remove when OCI adoption is complete, all servers will be reconciled
    94  	// regardless of labels at that point
    95  	palletAnnos := map[string]string{
    96  		pallet.KnnotationName:     "prometheus",
    97  		pallet.KnnotationTeam:     "ncrvoyix-swt-retail/edge-o11y",
    98  		pallet.KnnotationRevision: "c3918f",
    99  		pallet.KnnotationVersion:  "0.3.7",
   100  	}
   101  	p := s.prometheus(monitoringv1.PrometheusSpec{})
   102  	p.Annotations = palletAnnos
   103  	p.Spec.ExternalLabels = map[string]string{}
   104  	s.NoError(s.Client.Create(s.ctx, p))
   105  	s.Log("created prometheus", eclient.ObjectKeyFromObject(p))
   106  	s.testContainer(false)
   107  	s.testVolumes()
   108  
   109  	// test that it also works when container exists
   110  	s.NoError(s.Client.Delete(s.ctx, p))
   111  	p = s.prometheus(monitoringv1.PrometheusSpec{CommonPrometheusFields: monitoringv1.CommonPrometheusFields{
   112  		Containers: []corev1.Container{{
   113  			Name: "prometheus",
   114  		}},
   115  	}})
   116  	p.Annotations = palletAnnos
   117  	p.Spec.ExternalLabels = map[string]string{}
   118  	s.NoError(s.Client.Create(s.ctx, p))
   119  	s.testContainer(false)
   120  	s.testVolumes()
   121  
   122  	// clean up prometheus object
   123  	s.NoError(s.Client.Delete(s.ctx, p))
   124  }
   125  
   126  func (s *Suite) testContainer(legacy bool) {
   127  	var pcontainer corev1.Container
   128  	s.Eventually(func() bool {
   129  		p := &monitoringv1.Prometheus{}
   130  		if err := s.Client.Get(s.ctx, eclient.ObjectKeyFromRef(s.cfg.prometheus), p); err != nil {
   131  			return false
   132  		}
   133  
   134  		var err error
   135  		pcontainer, err = prometheusContainer(p.Spec.Containers)
   136  
   137  		return err == nil && len(pcontainer.Args) > 0
   138  	}, s.timeout, s.tick)
   139  
   140  	found := make(map[string]bool, len(s.expectedMetrics))
   141  	for _, metric := range s.expectedMetrics {
   142  		found[metric] = false
   143  		for _, arg := range pcontainer.Args {
   144  			if arg == metricArg(metric) {
   145  				found[metric] = true
   146  			}
   147  		}
   148  	}
   149  
   150  	for k, v := range found {
   151  		s.True(v, "expected metric %s not found", k)
   152  	}
   153  
   154  	s.Equal(dedupe(pcontainer.Args), pcontainer.Args)
   155  
   156  	var (
   157  		webEnableLifecycle = false
   158  		configFile         = false
   159  	)
   160  	for _, a := range pcontainer.Args {
   161  		switch a {
   162  		case "--config.file=/etc/prometheus/config_out/prometheus.env.yaml":
   163  			configFile = true
   164  		case "--web.enable-lifecycle":
   165  			webEnableLifecycle = true
   166  		}
   167  	}
   168  
   169  	s.True(webEnableLifecycle, "lifecycle argument not present")
   170  	s.True(configFile, "config file argument not present")
   171  
   172  	if legacy || s.cfg.clusterProvider != cluster.GKE {
   173  		s.Equal(envVars(), pcontainer.Env)
   174  	}
   175  }
   176  
   177  func (s *Suite) testVolumes() {
   178  	if s.cfg.clusterProvider == cluster.GKE {
   179  		return
   180  	}
   181  
   182  	var p monitoringv1.Prometheus
   183  	s.Eventually(func() bool {
   184  		if err := s.Client.Get(s.ctx, eclient.ObjectKeyFromRef(s.cfg.prometheus), &p); err != nil {
   185  			return false
   186  		}
   187  		return len(p.Spec.Volumes) > 0 && len(p.Spec.VolumeMounts) > 0
   188  	}, s.timeout, s.tick)
   189  
   190  	vols, mounts := volumeCfg()
   191  	s.Equal(vols, p.Spec.Volumes)
   192  	s.Equal(mounts, p.Spec.VolumeMounts)
   193  }
   194  
   195  func (s *Suite) prometheus(spec monitoringv1.PrometheusSpec) *monitoringv1.Prometheus {
   196  	return &monitoringv1.Prometheus{
   197  		ObjectMeta: metav1.ObjectMeta{
   198  			Name:      s.cfg.prometheus.Name,
   199  			Namespace: s.cfg.prometheus.Namespace,
   200  		},
   201  		Spec: spec,
   202  	}
   203  }
   204  
   205  func (s *Suite) setup(_ *framework.Framework) {
   206  	// create prometheus namespace
   207  	s.Require().NoError(s.Client.Create(s.ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{
   208  		Name: s.cfg.prometheus.Namespace,
   209  	}}))
   210  	s.Log("created namespace", s.cfg.prometheus.Namespace)
   211  	// schedule monitor resources
   212  	for _, o := range monitors {
   213  		c := o.DeepCopy()
   214  		c.SetNamespace(s.cfg.prometheus.Namespace)
   215  		s.Require().NoError(s.Client.Create(s.ctx, c))
   216  		s.Log("created monitor", c.GetName(), c.GetNamespace())
   217  	}
   218  }
   219  
   220  func (s *Suite) teardown(_ *framework.Framework) {
   221  	s.Require().NoError(s.Client.Delete(s.ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{
   222  		Name: s.cfg.prometheus.Namespace,
   223  	}}))
   224  }
   225  
   226  // buildSuite creats and returns a suite so we can more easily run unit tests
   227  // for multiple provider types
   228  func buildSuite(provider cluster.Type) *Suite {
   229  	cfg := &config{
   230  		reconcileInterval: time.Millisecond * 5,
   231  		prometheus: meta.NamespacedObjectReference{
   232  			Name:      fmt.Sprintf("prometheus-%s", provider),
   233  			Namespace: fmt.Sprintf("prometheus-%s", provider),
   234  		},
   235  		clusterProvider: string(provider),
   236  	}
   237  	mgr, err := create(
   238  		cfg,
   239  		controller.WithCfg(testEnv.Config),
   240  		controller.WithMetricsAddress("0"),
   241  	)
   242  	test.NoError(err)
   243  
   244  	// initialize frameworks now that we have the required config
   245  	k := k8s.New(testEnv.Config, k8s.WithCtrlManager(mgr))
   246  	f := framework.New("prometheusctl").
   247  		Component("prometheusctl").
   248  		Register(k)
   249  
   250  	// initialize suite
   251  	s := &Suite{
   252  		Framework:       f,
   253  		K8s:             k,
   254  		ctx:             context.Background(),
   255  		timeout:         1 * time.Second,
   256  		tick:            10 * time.Millisecond,
   257  		expectedMetrics: strings.Split(strings.TrimSpace(expectedMetricsData), "\n"),
   258  		cfg:             *cfg,
   259  	}
   260  
   261  	// register suite-specific lifecycle funcs
   262  	f.Setup(s.setup)
   263  	f.Teardown(s.teardown)
   264  
   265  	return s
   266  }
   267  
   268  func TestParseMetricsAnnotations(t *testing.T) {
   269  	tcs := map[string]struct {
   270  		meta     metav1.ObjectMeta
   271  		expected map[string]bool
   272  	}{
   273  		"simple": {
   274  			meta: objMeta(map[string]string{allowedMetrics: `
   275  kube_node_info
   276  kube_node_labels
   277  {__name__=~"kube_pod_.*"}
   278  `,
   279  			}),
   280  			expected: map[string]bool{"kube_node_info": true, "kube_node_labels": true, `{__name__=~"kube_pod_.*"}`: true},
   281  		},
   282  		"empty annotation": {
   283  			meta:     objMeta(map[string]string{allowedMetrics: ""}),
   284  			expected: map[string]bool{},
   285  		},
   286  		"no annotation": {
   287  			meta:     objMeta(map[string]string{}),
   288  			expected: map[string]bool{},
   289  		},
   290  	}
   291  
   292  	for name, tc := range tcs {
   293  		t.Run(name, func(t *testing.T) {
   294  			r := &PrometheusStackdriverReconciler{
   295  				metricsList: make(map[string]bool),
   296  			}
   297  			r.fetchMetricsAnnotations(tc.meta)
   298  			assert.Equal(t, tc.expected, r.metricsList)
   299  		})
   300  	}
   301  }
   302  
   303  func objMeta(annos map[string]string) metav1.ObjectMeta {
   304  	return metav1.ObjectMeta{
   305  		Annotations: annos,
   306  	}
   307  }
   308
View as plain text