filters_test.go

Documentation: sigs.k8s.io/controller-runtime/pkg/metrics/filters

     1  /*
     2  Copyright 2023 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package filters
    18  
    19  import (
    20  	"context"
    21  	"crypto/tls"
    22  	"errors"
    23  	"fmt"
    24  	"io"
    25  	"net/http"
    26  	"reflect"
    27  	"time"
    28  
    29  	. "github.com/onsi/ginkgo/v2"
    30  	. "github.com/onsi/gomega"
    31  	"github.com/prometheus/client_golang/prometheus"
    32  	authenticationv1 "k8s.io/api/authentication/v1"
    33  	corev1 "k8s.io/api/core/v1"
    34  	rbacv1 "k8s.io/api/rbac/v1"
    35  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    36  	"k8s.io/client-go/rest"
    37  	"k8s.io/utils/ptr"
    38  	"sigs.k8s.io/controller-runtime/pkg/client"
    39  	"sigs.k8s.io/controller-runtime/pkg/manager"
    40  	"sigs.k8s.io/controller-runtime/pkg/metrics"
    41  	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
    42  )
    43  
    44  var _ = Describe("manger.Manager", func() {
    45  	Describe("Start", func() {
    46  		Context("should start serving metrics with https and authn/authz", func() {
    47  			var srv metricsserver.Server
    48  			var defaultServer metricsDefaultServer
    49  			var opts manager.Options
    50  			var httpClient *http.Client
    51  
    52  			BeforeEach(func() {
    53  				srv = nil
    54  				newMetricsServer := func(options metricsserver.Options, config *rest.Config, httpClient *http.Client) (metricsserver.Server, error) {
    55  					var err error
    56  					srv, err = metricsserver.NewServer(options, config, httpClient)
    57  					if srv != nil {
    58  						defaultServer = srv.(metricsDefaultServer)
    59  					}
    60  					return srv, err
    61  				}
    62  				opts = manager.Options{
    63  					Metrics: metricsserver.Options{
    64  						BindAddress:    ":0",
    65  						SecureServing:  true,
    66  						FilterProvider: WithAuthenticationAndAuthorization,
    67  					},
    68  				}
    69  				v := reflect.ValueOf(&opts).Elem()
    70  				newMetricsField := v.FieldByName("newMetricsServer")
    71  				reflect.NewAt(newMetricsField.Type(), newMetricsField.Addr().UnsafePointer()).
    72  					Elem().
    73  					Set(reflect.ValueOf(newMetricsServer))
    74  				httpClient = &http.Client{Transport: &http.Transport{
    75  					TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec
    76  				}}
    77  			})
    78  
    79  			It("should serve metrics in its registry", func() {
    80  				one := prometheus.NewCounter(prometheus.CounterOpts{
    81  					Name: "test_one",
    82  					Help: "test metric for testing",
    83  				})
    84  				one.Inc()
    85  				err := metrics.Registry.Register(one)
    86  				Expect(err).NotTo(HaveOccurred())
    87  
    88  				m, err := manager.New(cfg, opts)
    89  				Expect(err).NotTo(HaveOccurred())
    90  
    91  				ctx, cancel := context.WithCancel(context.Background())
    92  				defer cancel()
    93  				go func() {
    94  					defer GinkgoRecover()
    95  					Expect(m.Start(ctx)).NotTo(HaveOccurred())
    96  				}()
    97  				<-m.Elected()
    98  				// Note: Wait until metrics server has been started. A finished leader election
    99  				// doesn't guarantee that the metrics server is up.
   100  				Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty())
   101  
   102  				// Setup service account with rights to "/metrics"
   103  				token, cleanup, err := setupServiceAccountForURL(ctx, m.GetClient(), "/metrics")
   104  				defer cleanup()
   105  				Expect(err).ToNot(HaveOccurred())
   106  
   107  				// GET /metrics with token.
   108  				metricsEndpoint := fmt.Sprintf("https://%s/metrics", defaultServer.GetBindAddr())
   109  				req, err := http.NewRequest("GET", metricsEndpoint, nil)
   110  				Expect(err).NotTo(HaveOccurred())
   111  				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
   112  				resp, err := httpClient.Do(req)
   113  				Expect(err).NotTo(HaveOccurred())
   114  				defer resp.Body.Close()
   115  				// This is expected as the token has rights for /metrics.
   116  				Expect(resp.StatusCode).To(Equal(200))
   117  
   118  				data, err := io.ReadAll(resp.Body)
   119  				Expect(err).NotTo(HaveOccurred())
   120  				Expect(string(data)).To(ContainSubstring("%s\n%s\n%s\n",
   121  					`# HELP test_one test metric for testing`,
   122  					`# TYPE test_one counter`,
   123  					`test_one 1`,
   124  				))
   125  
   126  				// Unregister will return false if the metric was never registered
   127  				ok := metrics.Registry.Unregister(one)
   128  				Expect(ok).To(BeTrue())
   129  			})
   130  
   131  			It("should serve extra endpoints", func() {
   132  				opts.Metrics.ExtraHandlers = map[string]http.Handler{
   133  					"/debug": http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
   134  						_, _ = w.Write([]byte("Some debug info"))
   135  					}),
   136  				}
   137  				m, err := manager.New(cfg, opts)
   138  				Expect(err).NotTo(HaveOccurred())
   139  
   140  				ctx, cancel := context.WithCancel(context.Background())
   141  				defer cancel()
   142  				go func() {
   143  					defer GinkgoRecover()
   144  					Expect(m.Start(ctx)).NotTo(HaveOccurred())
   145  				}()
   146  				<-m.Elected()
   147  				// Note: Wait until metrics server has been started. A finished leader election
   148  				// doesn't guarantee that the metrics server is up.
   149  				Eventually(func() string { return defaultServer.GetBindAddr() }, 10*time.Second).ShouldNot(BeEmpty())
   150  
   151  				// Setup service account with rights to "/debug"
   152  				token, cleanup, err := setupServiceAccountForURL(ctx, m.GetClient(), "/debug")
   153  				defer cleanup()
   154  				Expect(err).ToNot(HaveOccurred())
   155  
   156  				// GET /debug without token.
   157  				endpoint := fmt.Sprintf("https://%s/debug", defaultServer.GetBindAddr())
   158  				req, err := http.NewRequest("GET", endpoint, nil)
   159  				Expect(err).NotTo(HaveOccurred())
   160  				resp, err := httpClient.Do(req)
   161  				Expect(err).NotTo(HaveOccurred())
   162  				defer resp.Body.Close()
   163  				// This is expected as we didn't send a token.
   164  				Expect(resp.StatusCode).To(Equal(401))
   165  				body, err := io.ReadAll(resp.Body)
   166  				Expect(err).NotTo(HaveOccurred())
   167  				Expect(string(body)).To(ContainSubstring("Unauthorized"))
   168  
   169  				// PUT /debug with token.
   170  				req, err = http.NewRequest("PUT", endpoint, nil)
   171  				Expect(err).NotTo(HaveOccurred())
   172  				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
   173  				resp, err = httpClient.Do(req)
   174  				Expect(err).NotTo(HaveOccurred())
   175  				defer resp.Body.Close()
   176  				// This is expected as the token has rights for /debug.
   177  				Expect(resp.StatusCode).To(Equal(200))
   178  				body, err = io.ReadAll(resp.Body)
   179  				Expect(err).NotTo(HaveOccurred())
   180  				Expect(string(body)).To(Equal("Some debug info"))
   181  
   182  				// GET /metrics with token (but token only has rights for /debug).
   183  				metricsEndpoint := fmt.Sprintf("https://%s/metrics", defaultServer.GetBindAddr())
   184  				req, err = http.NewRequest("GET", metricsEndpoint, nil)
   185  				Expect(err).NotTo(HaveOccurred())
   186  				req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token))
   187  				resp, err = httpClient.Do(req)
   188  				Expect(err).NotTo(HaveOccurred())
   189  				defer resp.Body.Close()
   190  				Expect(resp.StatusCode).To(Equal(403))
   191  				body, err = io.ReadAll(resp.Body)
   192  				Expect(err).NotTo(HaveOccurred())
   193  				// Authorization denied is expected as the token only has rights for /debug not for /metrics.
   194  				Expect(string(body)).To(ContainSubstring("Authorization denied for user system:serviceaccount:default:metrics-test"))
   195  			})
   196  		})
   197  	})
   198  })
   199  
   200  type metricsDefaultServer interface {
   201  	GetBindAddr() string
   202  }
   203  
   204  func setupServiceAccountForURL(ctx context.Context, c client.Client, path string) (string, func(), error) {
   205  	createdObjects := []client.Object{}
   206  	cleanup := func() {
   207  		for _, obj := range createdObjects {
   208  			_ = c.Delete(ctx, obj)
   209  		}
   210  	}
   211  
   212  	sa := &corev1.ServiceAccount{
   213  		ObjectMeta: metav1.ObjectMeta{
   214  			Name:      "metrics-test",
   215  			Namespace: metav1.NamespaceDefault,
   216  		},
   217  	}
   218  	if err := c.Create(ctx, sa); err != nil {
   219  		return "", cleanup, err
   220  	}
   221  	createdObjects = append(createdObjects, sa)
   222  
   223  	cr := &rbacv1.ClusterRole{
   224  		ObjectMeta: metav1.ObjectMeta{
   225  			Name: "metrics-test",
   226  		},
   227  		Rules: []rbacv1.PolicyRule{
   228  			{
   229  				Verbs:           []string{"get", "put"},
   230  				NonResourceURLs: []string{path},
   231  			},
   232  		},
   233  	}
   234  	if err := c.Create(ctx, cr); err != nil {
   235  		return "", cleanup, err
   236  	}
   237  	createdObjects = append(createdObjects, cr)
   238  
   239  	crb := &rbacv1.ClusterRoleBinding{
   240  		ObjectMeta: metav1.ObjectMeta{
   241  			Name: "metrics-test",
   242  		},
   243  		Subjects: []rbacv1.Subject{
   244  			{
   245  				Kind:      rbacv1.ServiceAccountKind,
   246  				Name:      "metrics-test",
   247  				Namespace: metav1.NamespaceDefault,
   248  			},
   249  		},
   250  		RoleRef: rbacv1.RoleRef{
   251  			APIGroup: rbacv1.GroupName,
   252  			Kind:     "ClusterRole",
   253  			Name:     "metrics-test",
   254  		},
   255  	}
   256  	if err := c.Create(ctx, crb); err != nil {
   257  		return "", cleanup, err
   258  	}
   259  	createdObjects = append(createdObjects, crb)
   260  
   261  	tokenRequest := &authenticationv1.TokenRequest{
   262  		Spec: authenticationv1.TokenRequestSpec{
   263  			ExpirationSeconds: ptr.To(int64(2 * 60 * 60)), // 2 hours.
   264  		},
   265  	}
   266  	if err := c.SubResource("token").Create(ctx, sa, tokenRequest); err != nil {
   267  		return "", cleanup, err
   268  	}
   269  
   270  	if tokenRequest.Status.Token == "" {
   271  		return "", cleanup, errors.New("failed to get ServiceAccount token: token should not be empty")
   272  	}
   273  
   274  	return tokenRequest.Status.Token, cleanup, nil
   275  }
   276
View as plain text