podsecurity_test.go

Documentation: k8s.io/kubernetes/test/integration/auth

     1  /*
     2  Copyright 2021 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package auth
    18  
    19  import (
    20  	"context"
    21  	"crypto/tls"
    22  	"fmt"
    23  	"io"
    24  	"net"
    25  	"net/http"
    26  	"net/url"
    27  	"testing"
    28  	"time"
    29  
    30  	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
    31  	corev1 "k8s.io/api/core/v1"
    32  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    33  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    34  	"k8s.io/apimachinery/pkg/util/wait"
    35  	apiserver "k8s.io/apiserver/pkg/server"
    36  	"k8s.io/apiserver/pkg/server/dynamiccertificates"
    37  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    38  	"k8s.io/client-go/kubernetes"
    39  	"k8s.io/client-go/rest"
    40  	"k8s.io/component-base/featuregate"
    41  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    42  	"k8s.io/component-base/metrics/testutil"
    43  	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
    44  	"k8s.io/kubernetes/pkg/capabilities"
    45  	"k8s.io/kubernetes/pkg/features"
    46  	"k8s.io/kubernetes/test/integration/framework"
    47  	utiltest "k8s.io/kubernetes/test/utils"
    48  	podsecurityconfigloader "k8s.io/pod-security-admission/admission/api/load"
    49  	podsecurityserver "k8s.io/pod-security-admission/cmd/webhook/server"
    50  	podsecuritytest "k8s.io/pod-security-admission/test"
    51  )
    52  
    53  func TestPodSecurity(t *testing.T) {
    54  	// Enable all feature gates needed to allow all fields to be exercised
    55  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ProcMountType, true)()
    56  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.UserNamespacesSupport, true)()
    57  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AppArmor, true)()
    58  	// Start server
    59  	server := startPodSecurityServer(t)
    60  	opts := podsecuritytest.Options{
    61  		ClientConfig: server.ClientConfig,
    62  
    63  		// Don't pass in feature-gate info, so all testcases run
    64  
    65  		// TODO
    66  		ExemptClient:         nil,
    67  		ExemptNamespaces:     []string{},
    68  		ExemptRuntimeClasses: []string{},
    69  	}
    70  	podsecuritytest.Run(t, opts)
    71  
    72  	ValidatePluginMetrics(t, opts.ClientConfig)
    73  }
    74  
    75  // TestPodSecurityGAOnly ensures policies pass with only GA features enabled
    76  func TestPodSecurityGAOnly(t *testing.T) {
    77  	// Disable all alpha and beta features
    78  	for k, v := range utilfeature.DefaultFeatureGate.DeepCopy().GetAll() {
    79  		if k == "AllAlpha" || k == "AllBeta" {
    80  			// Skip special features. When processed first, special features may
    81  			// erroneously disable other features.
    82  			continue
    83  		} else if v.PreRelease == featuregate.Alpha || v.PreRelease == featuregate.Beta {
    84  			defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, k, false)()
    85  		}
    86  	}
    87  	// Start server
    88  	server := startPodSecurityServer(t)
    89  
    90  	opts := podsecuritytest.Options{
    91  		ClientConfig: server.ClientConfig,
    92  		// Pass in feature gate info so negative test cases depending on alpha or beta features can be skipped
    93  		Features: utilfeature.DefaultFeatureGate,
    94  	}
    95  	podsecuritytest.Run(t, opts)
    96  
    97  	ValidatePluginMetrics(t, opts.ClientConfig)
    98  }
    99  
   100  func TestPodSecurityWebhook(t *testing.T) {
   101  	// Enable all feature gates needed to allow all fields to be exercised
   102  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ProcMountType, true)()
   103  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.UserNamespacesSupport, true)()
   104  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.AppArmor, true)()
   105  
   106  	// Start test API server.
   107  	capabilities.SetForTests(capabilities.Capabilities{AllowPrivileged: true})
   108  	testServer := kubeapiservertesting.StartTestServerOrDie(t, kubeapiservertesting.NewDefaultTestServerOptions(), []string{
   109  		"--anonymous-auth=false",
   110  		"--allow-privileged=true",
   111  		// The webhook should pass tests even when PodSecurity is disabled.
   112  		"--disable-admission-plugins=PodSecurity",
   113  	}, framework.SharedEtcd())
   114  	t.Cleanup(testServer.TearDownFn)
   115  
   116  	webhookAddr, err := startPodSecurityWebhook(t, testServer)
   117  	if err != nil {
   118  		t.Fatalf("Failed to start webhook server: %v", err)
   119  	}
   120  	if err := installWebhook(t, testServer.ClientConfig, webhookAddr); err != nil {
   121  		t.Fatalf("Failed to install webhook configuration: %v", err)
   122  	}
   123  
   124  	opts := podsecuritytest.Options{
   125  		ClientConfig: testServer.ClientConfig,
   126  
   127  		// Don't pass in feature-gate info, so all testcases run
   128  
   129  		// TODO
   130  		ExemptClient:         nil,
   131  		ExemptNamespaces:     []string{},
   132  		ExemptRuntimeClasses: []string{},
   133  	}
   134  	podsecuritytest.Run(t, opts)
   135  
   136  	ValidateWebhookMetrics(t, webhookAddr)
   137  }
   138  
   139  func startPodSecurityServer(t *testing.T) *kubeapiservertesting.TestServer {
   140  	// ensure the global is set to allow privileged containers
   141  	capabilities.SetForTests(capabilities.Capabilities{AllowPrivileged: true})
   142  
   143  	server := kubeapiservertesting.StartTestServerOrDie(t, kubeapiservertesting.NewDefaultTestServerOptions(), []string{
   144  		"--anonymous-auth=false",
   145  		"--enable-admission-plugins=PodSecurity",
   146  		"--allow-privileged=true",
   147  		// TODO: "--admission-control-config-file=" + admissionConfigFile.Name(),
   148  	}, framework.SharedEtcd())
   149  	t.Cleanup(server.TearDownFn)
   150  	return server
   151  }
   152  
   153  func startPodSecurityWebhook(t *testing.T, testServer *kubeapiservertesting.TestServer) (addr string, err error) {
   154  	// listener, port, err := apiserver.CreateListener("tcp", "127.0.0.1:", net.ListenConfig{})
   155  	secureListener, err := net.Listen("tcp", "127.0.0.1:")
   156  	if err != nil {
   157  		return "", err
   158  	}
   159  	insecureListener, err := net.Listen("tcp", "127.0.0.1:")
   160  	if err != nil {
   161  		return "", err
   162  	}
   163  	cert, err := dynamiccertificates.NewStaticCertKeyContent("localhost", utiltest.LocalhostCert, utiltest.LocalhostKey)
   164  	if err != nil {
   165  		return "", err
   166  	}
   167  	defaultConfig, err := podsecurityconfigloader.LoadFromData(nil) // load the default
   168  	if err != nil {
   169  		return "", err
   170  	}
   171  
   172  	c := podsecurityserver.Config{
   173  		SecureServing: &apiserver.SecureServingInfo{
   174  			Listener: secureListener,
   175  			Cert:     cert,
   176  		},
   177  		InsecureServing: &apiserver.DeprecatedInsecureServingInfo{
   178  			Listener: insecureListener,
   179  		},
   180  		KubeConfig:        testServer.ClientConfig,
   181  		PodSecurityConfig: defaultConfig,
   182  	}
   183  
   184  	t.Logf("Starting webhook server...")
   185  	webhookServer, err := podsecurityserver.Setup(&c)
   186  	if err != nil {
   187  		return "", err
   188  	}
   189  
   190  	ctx, cancel := context.WithCancel(context.Background())
   191  	go webhookServer.Start(ctx)
   192  	t.Cleanup(cancel)
   193  
   194  	// Wait for server to be ready
   195  	t.Logf("Waiting for webhook server /readyz to be ok...")
   196  	readyz := (&url.URL{
   197  		Scheme: "http",
   198  		Host:   c.InsecureServing.Listener.Addr().String(),
   199  		Path:   "/readyz",
   200  	}).String()
   201  	if err := wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
   202  		resp, err := http.Get(readyz)
   203  		if err != nil {
   204  			return false, err
   205  		}
   206  		defer resp.Body.Close()
   207  		return resp.StatusCode == 200, nil
   208  	}); err != nil {
   209  		return "", err
   210  	}
   211  
   212  	return c.SecureServing.Listener.Addr().String(), nil
   213  }
   214  
   215  func installWebhook(t *testing.T, clientConfig *rest.Config, addr string) error {
   216  	client, err := kubernetes.NewForConfig(clientConfig)
   217  	if err != nil {
   218  		return fmt.Errorf("error creating client: %w", err)
   219  	}
   220  
   221  	fail := admissionregistrationv1.Fail
   222  	equivalent := admissionregistrationv1.Equivalent
   223  	none := admissionregistrationv1.SideEffectClassNone
   224  	endpoint := (&url.URL{
   225  		Scheme: "https",
   226  		Host:   addr,
   227  	}).String()
   228  
   229  	// Installing Admission webhook to API server
   230  	_, err = client.AdmissionregistrationV1().ValidatingWebhookConfigurations().Create(context.TODO(), &admissionregistrationv1.ValidatingWebhookConfiguration{
   231  		ObjectMeta: metav1.ObjectMeta{Name: "podsecurity-webhook.integration.test"},
   232  		Webhooks: []admissionregistrationv1.ValidatingWebhook{
   233  			{
   234  				Name: "podsecurity-webhook.integration.test",
   235  				ClientConfig: admissionregistrationv1.WebhookClientConfig{
   236  					URL:      &endpoint,
   237  					CABundle: utiltest.LocalhostCert,
   238  				},
   239  				Rules: []admissionregistrationv1.RuleWithOperations{
   240  					{
   241  						Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update},
   242  						Rule: admissionregistrationv1.Rule{
   243  							APIGroups:   []string{""},
   244  							APIVersions: []string{"v1"},
   245  							Resources:   []string{"namespaces", "pods", "pods/ephemeralcontainers", "replicationcontrollers", "podtemplates"},
   246  						},
   247  					},
   248  					{
   249  						Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update},
   250  						Rule: admissionregistrationv1.Rule{
   251  							APIGroups:   []string{"apps"},
   252  							APIVersions: []string{"v1"},
   253  							Resources:   []string{"replicasets", "deployments", "statefulsets", "daemonsets"},
   254  						},
   255  					},
   256  					{
   257  						Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create, admissionregistrationv1.Update},
   258  						Rule: admissionregistrationv1.Rule{
   259  							APIGroups:   []string{"batch"},
   260  							APIVersions: []string{"v1"},
   261  							Resources:   []string{"cronjobs", "jobs"},
   262  						},
   263  					},
   264  				},
   265  				FailurePolicy:           &fail,
   266  				MatchPolicy:             &equivalent,
   267  				AdmissionReviewVersions: []string{"v1"},
   268  				SideEffects:             &none,
   269  			},
   270  		},
   271  	}, metav1.CreateOptions{})
   272  	if err != nil {
   273  		return err
   274  	}
   275  
   276  	t.Logf("Waiting for webhook to be established...")
   277  	invalidNamespace := &corev1.Namespace{
   278  		ObjectMeta: metav1.ObjectMeta{
   279  			Name: "validation-fail",
   280  			Labels: map[string]string{
   281  				"pod-security.kubernetes.io/enforce": "invalid",
   282  			},
   283  		},
   284  	}
   285  	// Wait for the invalid namespace to be rejected.
   286  	if err := wait.PollImmediate(100*time.Millisecond, wait.ForeverTestTimeout, func() (bool, error) {
   287  		_, err := client.CoreV1().Namespaces().Create(context.TODO(), invalidNamespace, metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}})
   288  		if err != nil && apierrors.IsInvalid(err) {
   289  			return true, nil // An Invalid error indicates the webhook rejected the invalid level.
   290  		}
   291  		return false, nil
   292  	}); err != nil {
   293  		return err
   294  	}
   295  
   296  	return nil
   297  }
   298  
   299  func ValidatePluginMetrics(t *testing.T, clientConfig *rest.Config) {
   300  	client, err := kubernetes.NewForConfig(clientConfig)
   301  	if err != nil {
   302  		t.Fatalf("Error creating client: %v", err)
   303  	}
   304  	ctx := context.Background()
   305  	data, err := client.CoreV1().RESTClient().Get().AbsPath("metrics").DoRaw(ctx)
   306  	if err != nil {
   307  		t.Fatalf("Failed to read metrics: %v", err)
   308  	}
   309  	validateMetrics(t, data)
   310  }
   311  
   312  func ValidateWebhookMetrics(t *testing.T, webhookAddr string) {
   313  	endpoint := &url.URL{
   314  		Scheme: "https",
   315  		Host:   webhookAddr,
   316  		Path:   "/metrics",
   317  	}
   318  	client := &http.Client{Transport: &http.Transport{
   319  		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
   320  	}}
   321  	resp, err := client.Get(endpoint.String())
   322  	if err != nil {
   323  		t.Fatalf("Failed to fetch metrics from %s: %v", endpoint.String(), err)
   324  	}
   325  	defer resp.Body.Close()
   326  	if resp.StatusCode != http.StatusOK {
   327  		t.Fatalf("Non-200 response trying to scrape metrics from %s: %v", endpoint.String(), resp)
   328  	}
   329  	data, err := io.ReadAll(resp.Body)
   330  	if err != nil {
   331  		t.Fatalf("Unable to read metrics response: %v", err)
   332  	}
   333  	validateMetrics(t, data)
   334  }
   335  
   336  func validateMetrics(t *testing.T, rawMetrics []byte) {
   337  	metrics := testutil.NewMetrics()
   338  	if err := testutil.ParseMetrics(string(rawMetrics), &metrics); err != nil {
   339  		t.Fatalf("Failed to parse metrics: %v", err)
   340  	}
   341  
   342  	if err := testutil.ValidateMetrics(metrics, "pod_security_evaluations_total",
   343  		"decision", "policy_level", "policy_version", "mode", "request_operation", "resource", "subresource"); err != nil {
   344  		t.Errorf("Metric validation failed: %v", err)
   345  	}
   346  	if err := testutil.ValidateMetrics(metrics, "pod_security_exemptions_total",
   347  		"request_operation", "resource", "subresource"); err != nil {
   348  		t.Errorf("Metric validation failed: %v", err)
   349  	}
   350  }
   351
View as plain text