certificates.go

Documentation: k8s.io/kubernetes/test/e2e/auth

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package auth
    18  
    19  import (
    20  	"context"
    21  	"crypto/x509"
    22  	"crypto/x509/pkix"
    23  	"encoding/json"
    24  	"encoding/pem"
    25  	"time"
    26  
    27  	"github.com/onsi/ginkgo/v2"
    28  	"github.com/onsi/gomega"
    29  
    30  	certificatesv1 "k8s.io/api/certificates/v1"
    31  	v1 "k8s.io/api/core/v1"
    32  	rbacv1 "k8s.io/api/rbac/v1"
    33  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    34  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    35  	"k8s.io/apimachinery/pkg/types"
    36  	"k8s.io/apimachinery/pkg/util/wait"
    37  	"k8s.io/apimachinery/pkg/watch"
    38  	"k8s.io/apiserver/pkg/server/dynamiccertificates"
    39  	certificatesclient "k8s.io/client-go/kubernetes/typed/certificates/v1"
    40  	"k8s.io/client-go/rest"
    41  	"k8s.io/client-go/util/cert"
    42  	"k8s.io/client-go/util/certificate/csr"
    43  	"k8s.io/kubernetes/test/e2e/framework"
    44  	"k8s.io/kubernetes/test/utils"
    45  	admissionapi "k8s.io/pod-security-admission/api"
    46  )
    47  
    48  var _ = SIGDescribe("Certificates API [Privileged:ClusterAdmin]", func() {
    49  	f := framework.NewDefaultFramework("certificates")
    50  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    51  
    52  	/*
    53  		Release: v1.19
    54  		Testname: CertificateSigningRequest API Client Certificate
    55  		Description:
    56  		The certificatesigningrequests resource must accept a request for a certificate signed by kubernetes.io/kube-apiserver-client.
    57  		The issued certificate must be valid as a client certificate used to authenticate to the kube-apiserver.
    58  	*/
    59  	ginkgo.It("should support building a client with a CSR", func(ctx context.Context) {
    60  		const commonName = "tester-csr"
    61  
    62  		csrClient := f.ClientSet.CertificatesV1().CertificateSigningRequests()
    63  
    64  		pk, err := utils.NewPrivateKey()
    65  		framework.ExpectNoError(err)
    66  
    67  		pkder := x509.MarshalPKCS1PrivateKey(pk)
    68  		pkpem := pem.EncodeToMemory(&pem.Block{
    69  			Type:  "RSA PRIVATE KEY",
    70  			Bytes: pkder,
    71  		})
    72  
    73  		csrb, err := cert.MakeCSR(pk, &pkix.Name{CommonName: commonName}, nil, nil)
    74  		framework.ExpectNoError(err)
    75  
    76  		csrTemplate := &certificatesv1.CertificateSigningRequest{
    77  			ObjectMeta: metav1.ObjectMeta{
    78  				GenerateName: commonName + "-",
    79  			},
    80  			Spec: certificatesv1.CertificateSigningRequestSpec{
    81  				Request: csrb,
    82  				Usages: []certificatesv1.KeyUsage{
    83  					certificatesv1.UsageDigitalSignature,
    84  					certificatesv1.UsageKeyEncipherment,
    85  					certificatesv1.UsageClientAuth,
    86  				},
    87  				SignerName:        certificatesv1.KubeAPIServerClientSignerName,
    88  				ExpirationSeconds: csr.DurationToExpirationSeconds(time.Hour),
    89  			},
    90  		}
    91  
    92  		// Grant permissions to the new user
    93  		clusterRole, err := f.ClientSet.RbacV1().ClusterRoles().Create(ctx, &rbacv1.ClusterRole{
    94  			ObjectMeta: metav1.ObjectMeta{GenerateName: commonName + "-"},
    95  			Rules:      []rbacv1.PolicyRule{{Verbs: []string{"create"}, APIGroups: []string{"certificates.k8s.io"}, Resources: []string{"certificatesigningrequests"}}},
    96  		}, metav1.CreateOptions{})
    97  		if err != nil {
    98  			// Tolerate RBAC not being enabled
    99  			framework.Logf("error granting permissions to %s, create certificatesigningrequests permissions must be granted out of band: %v", commonName, err)
   100  		} else {
   101  			defer func() {
   102  				framework.ExpectNoError(f.ClientSet.RbacV1().ClusterRoles().Delete(ctx, clusterRole.Name, metav1.DeleteOptions{}))
   103  			}()
   104  		}
   105  
   106  		clusterRoleBinding, err := f.ClientSet.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{
   107  			ObjectMeta: metav1.ObjectMeta{GenerateName: commonName + "-"},
   108  			RoleRef:    rbacv1.RoleRef{APIGroup: "rbac.authorization.k8s.io", Kind: "ClusterRole", Name: clusterRole.Name},
   109  			Subjects:   []rbacv1.Subject{{APIGroup: "rbac.authorization.k8s.io", Kind: "User", Name: commonName}},
   110  		}, metav1.CreateOptions{})
   111  		if err != nil {
   112  			// Tolerate RBAC not being enabled
   113  			framework.Logf("error granting permissions to %s, create certificatesigningrequests permissions must be granted out of band: %v", commonName, err)
   114  		} else {
   115  			defer func() {
   116  				framework.ExpectNoError(f.ClientSet.RbacV1().ClusterRoleBindings().Delete(ctx, clusterRoleBinding.Name, metav1.DeleteOptions{}))
   117  			}()
   118  		}
   119  
   120  		framework.Logf("creating CSR")
   121  		csr, err := csrClient.Create(ctx, csrTemplate, metav1.CreateOptions{})
   122  		framework.ExpectNoError(err)
   123  		defer func() {
   124  			framework.ExpectNoError(csrClient.Delete(ctx, csr.Name, metav1.DeleteOptions{}))
   125  		}()
   126  
   127  		framework.Logf("approving CSR")
   128  		framework.ExpectNoError(wait.Poll(5*time.Second, time.Minute, func() (bool, error) {
   129  			csr.Status.Conditions = []certificatesv1.CertificateSigningRequestCondition{
   130  				{
   131  					Type:    certificatesv1.CertificateApproved,
   132  					Status:  v1.ConditionTrue,
   133  					Reason:  "E2E",
   134  					Message: "Set from an e2e test",
   135  				},
   136  			}
   137  			csr, err = csrClient.UpdateApproval(ctx, csr.Name, csr, metav1.UpdateOptions{})
   138  			if err != nil {
   139  				csr, _ = csrClient.Get(ctx, csr.Name, metav1.GetOptions{})
   140  				framework.Logf("err updating approval: %v", err)
   141  				return false, nil
   142  			}
   143  			return true, nil
   144  		}))
   145  
   146  		framework.Logf("waiting for CSR to be signed")
   147  		framework.ExpectNoError(wait.Poll(5*time.Second, time.Minute, func() (bool, error) {
   148  			csr, err = csrClient.Get(ctx, csr.Name, metav1.GetOptions{})
   149  			if err != nil {
   150  				framework.Logf("error getting csr: %v", err)
   151  				return false, nil
   152  			}
   153  			if len(csr.Status.Certificate) == 0 {
   154  				framework.Logf("csr not signed yet")
   155  				return false, nil
   156  			}
   157  			return true, nil
   158  		}))
   159  
   160  		framework.Logf("testing the client")
   161  		rcfg, err := framework.LoadConfig()
   162  		framework.ExpectNoError(err)
   163  		rcfg = rest.AnonymousClientConfig(rcfg)
   164  		rcfg.TLSClientConfig.CertData = csr.Status.Certificate
   165  		rcfg.TLSClientConfig.KeyData = pkpem
   166  
   167  		certs, err := cert.ParseCertsPEM(csr.Status.Certificate)
   168  		framework.ExpectNoError(err)
   169  		gomega.Expect(certs).To(gomega.HaveLen(1), "expected a single cert, got %#v", certs)
   170  		cert := certs[0]
   171  		// make sure the cert is not valid for longer than our requested time (plus allowance for backdating)
   172  		if e, a := time.Hour+5*time.Minute, cert.NotAfter.Sub(cert.NotBefore); a > e {
   173  			framework.Failf("expected cert valid for %s or less, got %s: %s", e, a, dynamiccertificates.GetHumanCertDetail(cert))
   174  		}
   175  
   176  		newClient, err := certificatesclient.NewForConfig(rcfg)
   177  		framework.ExpectNoError(err)
   178  
   179  		framework.Logf("creating CSR as new client")
   180  		newCSR, err := newClient.CertificateSigningRequests().Create(ctx, csrTemplate, metav1.CreateOptions{})
   181  		framework.ExpectNoError(err)
   182  		defer func() {
   183  			framework.ExpectNoError(csrClient.Delete(ctx, newCSR.Name, metav1.DeleteOptions{}))
   184  		}()
   185  		gomega.Expect(newCSR.Spec.Username).To(gomega.Equal(commonName))
   186  	})
   187  
   188  	/*
   189  		Release: v1.19
   190  		Testname: CertificateSigningRequest API
   191  		Description:
   192  		The certificates.k8s.io API group MUST exists in the /apis discovery document.
   193  		The certificates.k8s.io/v1 API group/version MUST exist in the /apis/certificates.k8s.io discovery document.
   194  		The certificatesigningrequests, certificatesigningrequests/approval, and certificatesigningrequests/status
   195  		  resources MUST exist in the /apis/certificates.k8s.io/v1 discovery document.
   196  		The certificatesigningrequests resource must support create, get, list, watch, update, patch, delete, and deletecollection.
   197  		The certificatesigningrequests/approval resource must support get, update, patch.
   198  		The certificatesigningrequests/status resource must support get, update, patch.
   199  	*/
   200  	framework.ConformanceIt("should support CSR API operations", func(ctx context.Context) {
   201  
   202  		// Setup
   203  		csrVersion := "v1"
   204  		csrClient := f.ClientSet.CertificatesV1().CertificateSigningRequests()
   205  		csrResource := certificatesv1.SchemeGroupVersion.WithResource("certificatesigningrequests")
   206  
   207  		pk, err := utils.NewPrivateKey()
   208  		framework.ExpectNoError(err)
   209  
   210  		csrData, err := cert.MakeCSR(pk, &pkix.Name{CommonName: "e2e.example.com"}, []string{"e2e.example.com"}, nil)
   211  		framework.ExpectNoError(err)
   212  
   213  		certificateData, _, err := cert.GenerateSelfSignedCertKey("e2e.example.com", nil, []string{"e2e.example.com"})
   214  		framework.ExpectNoError(err)
   215  		certificateDataJSON, err := json.Marshal(certificateData)
   216  		framework.ExpectNoError(err)
   217  
   218  		signerName := "example.com/e2e-" + f.UniqueName
   219  		csrTemplate := &certificatesv1.CertificateSigningRequest{
   220  			ObjectMeta: metav1.ObjectMeta{GenerateName: "e2e-example-csr-"},
   221  			Spec: certificatesv1.CertificateSigningRequestSpec{
   222  				Request:           csrData,
   223  				SignerName:        signerName,
   224  				ExpirationSeconds: csr.DurationToExpirationSeconds(time.Hour),
   225  				Usages:            []certificatesv1.KeyUsage{certificatesv1.UsageDigitalSignature, certificatesv1.UsageKeyEncipherment, certificatesv1.UsageServerAuth},
   226  			},
   227  		}
   228  
   229  		// Discovery
   230  
   231  		ginkgo.By("getting /apis")
   232  		{
   233  			discoveryGroups, err := f.ClientSet.Discovery().ServerGroups()
   234  			framework.ExpectNoError(err)
   235  			found := false
   236  			for _, group := range discoveryGroups.Groups {
   237  				if group.Name == certificatesv1.GroupName {
   238  					for _, version := range group.Versions {
   239  						if version.Version == csrVersion {
   240  							found = true
   241  							break
   242  						}
   243  					}
   244  				}
   245  			}
   246  			if !found {
   247  				framework.Failf("expected certificates API group/version, got %#v", discoveryGroups.Groups)
   248  			}
   249  		}
   250  
   251  		ginkgo.By("getting /apis/certificates.k8s.io")
   252  		{
   253  			group := &metav1.APIGroup{}
   254  			err := f.ClientSet.Discovery().RESTClient().Get().AbsPath("/apis/certificates.k8s.io").Do(ctx).Into(group)
   255  			framework.ExpectNoError(err)
   256  			found := false
   257  			for _, version := range group.Versions {
   258  				if version.Version == csrVersion {
   259  					found = true
   260  					break
   261  				}
   262  			}
   263  			if !found {
   264  				framework.Failf("expected certificates API version, got %#v", group.Versions)
   265  			}
   266  		}
   267  
   268  		ginkgo.By("getting /apis/certificates.k8s.io/" + csrVersion)
   269  		{
   270  			resources, err := f.ClientSet.Discovery().ServerResourcesForGroupVersion(certificatesv1.SchemeGroupVersion.String())
   271  			framework.ExpectNoError(err)
   272  			foundCSR, foundApproval, foundStatus := false, false, false
   273  			for _, resource := range resources.APIResources {
   274  				switch resource.Name {
   275  				case "certificatesigningrequests":
   276  					foundCSR = true
   277  				case "certificatesigningrequests/approval":
   278  					foundApproval = true
   279  				case "certificatesigningrequests/status":
   280  					foundStatus = true
   281  				}
   282  			}
   283  			if !foundCSR {
   284  				framework.Failf("expected certificatesigningrequests, got %#v", resources.APIResources)
   285  			}
   286  			if !foundApproval {
   287  				framework.Failf("expected certificatesigningrequests/approval, got %#v", resources.APIResources)
   288  			}
   289  			if !foundStatus {
   290  				framework.Failf("expected certificatesigningrequests/status, got %#v", resources.APIResources)
   291  			}
   292  		}
   293  
   294  		// Main resource create/read/update/watch operations
   295  
   296  		ginkgo.By("creating")
   297  		_, err = csrClient.Create(ctx, csrTemplate, metav1.CreateOptions{})
   298  		framework.ExpectNoError(err)
   299  		_, err = csrClient.Create(ctx, csrTemplate, metav1.CreateOptions{})
   300  		framework.ExpectNoError(err)
   301  		createdCSR, err := csrClient.Create(ctx, csrTemplate, metav1.CreateOptions{})
   302  		framework.ExpectNoError(err)
   303  
   304  		ginkgo.By("getting")
   305  		gottenCSR, err := csrClient.Get(ctx, createdCSR.Name, metav1.GetOptions{})
   306  		framework.ExpectNoError(err)
   307  		gomega.Expect(gottenCSR.UID).To(gomega.Equal(createdCSR.UID))
   308  		gomega.Expect(gottenCSR.Spec.ExpirationSeconds).To(gomega.Equal(csr.DurationToExpirationSeconds(time.Hour)))
   309  
   310  		ginkgo.By("listing")
   311  		csrs, err := csrClient.List(ctx, metav1.ListOptions{FieldSelector: "spec.signerName=" + signerName})
   312  		framework.ExpectNoError(err)
   313  		gomega.Expect(csrs.Items).To(gomega.HaveLen(3), "filtered list should have 3 items")
   314  
   315  		ginkgo.By("watching")
   316  		framework.Logf("starting watch")
   317  		csrWatch, err := csrClient.Watch(ctx, metav1.ListOptions{ResourceVersion: csrs.ResourceVersion, FieldSelector: "metadata.name=" + createdCSR.Name})
   318  		framework.ExpectNoError(err)
   319  
   320  		ginkgo.By("patching")
   321  		patchedCSR, err := csrClient.Patch(ctx, createdCSR.Name, types.MergePatchType, []byte(`{"metadata":{"annotations":{"patched":"true"}}}`), metav1.PatchOptions{})
   322  		framework.ExpectNoError(err)
   323  		gomega.Expect(patchedCSR.Annotations).To(gomega.HaveKeyWithValue("patched", "true"), "patched object should have the applied annotation")
   324  
   325  		ginkgo.By("updating")
   326  		csrToUpdate := patchedCSR.DeepCopy()
   327  		csrToUpdate.Annotations["updated"] = "true"
   328  		updatedCSR, err := csrClient.Update(ctx, csrToUpdate, metav1.UpdateOptions{})
   329  		framework.ExpectNoError(err)
   330  		gomega.Expect(updatedCSR.Annotations).To(gomega.HaveKeyWithValue("updated", "true"), "updated object should have the applied annotation")
   331  
   332  		framework.Logf("waiting for watch events with expected annotations")
   333  		for sawAnnotations := false; !sawAnnotations; {
   334  			select {
   335  			case evt, ok := <-csrWatch.ResultChan():
   336  				if !ok {
   337  					framework.Fail("watch channel should not close")
   338  				}
   339  				gomega.Expect(evt.Type).To(gomega.Equal(watch.Modified))
   340  				watchedCSR, isCSR := evt.Object.(*certificatesv1.CertificateSigningRequest)
   341  				if !isCSR {
   342  					framework.Failf("expected CSR, got %T", evt.Object)
   343  				}
   344  				if watchedCSR.Annotations["patched"] == "true" {
   345  					framework.Logf("saw patched and updated annotations")
   346  					sawAnnotations = true
   347  					csrWatch.Stop()
   348  				} else {
   349  					framework.Logf("missing expected annotations, waiting: %#v", watchedCSR.Annotations)
   350  				}
   351  			case <-time.After(wait.ForeverTestTimeout):
   352  				framework.Fail("timed out waiting for watch event")
   353  			}
   354  		}
   355  
   356  		// /approval subresource operations
   357  
   358  		ginkgo.By("getting /approval")
   359  		gottenApproval, err := f.DynamicClient.Resource(csrResource).Get(ctx, createdCSR.Name, metav1.GetOptions{}, "approval")
   360  		framework.ExpectNoError(err)
   361  		gomega.Expect(gottenApproval.GetObjectKind().GroupVersionKind()).To(gomega.Equal(certificatesv1.SchemeGroupVersion.WithKind("CertificateSigningRequest")))
   362  		gomega.Expect(gottenApproval.GetUID()).To(gomega.Equal(createdCSR.UID))
   363  
   364  		ginkgo.By("patching /approval")
   365  		patchedApproval, err := csrClient.Patch(ctx, createdCSR.Name, types.MergePatchType,
   366  			[]byte(`{"metadata":{"annotations":{"patchedapproval":"true"}},"status":{"conditions":[{"type":"ApprovalPatch","status":"True","reason":"e2e"}]}}`),
   367  			metav1.PatchOptions{}, "approval")
   368  		framework.ExpectNoError(err)
   369  		gomega.Expect(patchedApproval.Status.Conditions).To(gomega.HaveLen(1), "patched object should have the applied condition")
   370  		gomega.Expect(string(patchedApproval.Status.Conditions[0].Type)).To(gomega.Equal("ApprovalPatch"), "patched object should have the applied condition, got %#v", patchedApproval.Status.Conditions)
   371  		gomega.Expect(patchedApproval.Annotations).To(gomega.HaveKeyWithValue("patchedapproval", "true"), "patched object should have the applied annotation")
   372  
   373  		ginkgo.By("updating /approval")
   374  		approvalToUpdate := patchedApproval.DeepCopy()
   375  		approvalToUpdate.Status.Conditions = append(approvalToUpdate.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{
   376  			Type:    certificatesv1.CertificateApproved,
   377  			Status:  v1.ConditionTrue,
   378  			Reason:  "E2E",
   379  			Message: "Set from an e2e test",
   380  		})
   381  		updatedApproval, err := csrClient.UpdateApproval(ctx, approvalToUpdate.Name, approvalToUpdate, metav1.UpdateOptions{})
   382  		framework.ExpectNoError(err)
   383  		gomega.Expect(updatedApproval.Status.Conditions).To(gomega.HaveLen(2), "updated object should have the applied condition, got %#v", updatedApproval.Status.Conditions)
   384  		gomega.Expect(updatedApproval.Status.Conditions[1].Type).To(gomega.Equal(certificatesv1.CertificateApproved), "updated object should have the approved condition, got %#v", updatedApproval.Status.Conditions)
   385  
   386  		// /status subresource operations
   387  
   388  		ginkgo.By("getting /status")
   389  		gottenStatus, err := f.DynamicClient.Resource(csrResource).Get(ctx, createdCSR.Name, metav1.GetOptions{}, "status")
   390  		framework.ExpectNoError(err)
   391  		gomega.Expect(gottenStatus.GetObjectKind().GroupVersionKind()).To(gomega.Equal(certificatesv1.SchemeGroupVersion.WithKind("CertificateSigningRequest")))
   392  		gomega.Expect(gottenStatus.GetUID()).To(gomega.Equal(createdCSR.UID))
   393  
   394  		ginkgo.By("patching /status")
   395  		patchedStatus, err := csrClient.Patch(ctx, createdCSR.Name, types.MergePatchType,
   396  			[]byte(`{"metadata":{"annotations":{"patchedstatus":"true"}},"status":{"certificate":`+string(certificateDataJSON)+`}}`),
   397  			metav1.PatchOptions{}, "status")
   398  		framework.ExpectNoError(err)
   399  		gomega.Expect(patchedStatus.Status.Certificate).To(gomega.Equal(certificateData), "patched object should have the applied certificate")
   400  		gomega.Expect(patchedStatus.Annotations).To(gomega.HaveKeyWithValue("patchedstatus", "true"), "patched object should have the applied annotation")
   401  
   402  		ginkgo.By("updating /status")
   403  		statusToUpdate := patchedStatus.DeepCopy()
   404  		statusToUpdate.Status.Conditions = append(statusToUpdate.Status.Conditions, certificatesv1.CertificateSigningRequestCondition{
   405  			Type:    "StatusUpdate",
   406  			Status:  v1.ConditionTrue,
   407  			Reason:  "E2E",
   408  			Message: "Set from an e2e test",
   409  		})
   410  		updatedStatus, err := csrClient.UpdateStatus(ctx, statusToUpdate, metav1.UpdateOptions{})
   411  		framework.ExpectNoError(err)
   412  		gomega.Expect(updatedStatus.Status.Conditions).To(gomega.HaveLen(len(statusToUpdate.Status.Conditions)), "updated object should have the applied condition, got %#v", updatedStatus.Status.Conditions)
   413  		gomega.Expect(string(updatedStatus.Status.Conditions[len(updatedStatus.Status.Conditions)-1].Type)).To(gomega.Equal("StatusUpdate"), "updated object should have the approved condition, got %#v", updatedStatus.Status.Conditions)
   414  
   415  		// main resource delete operations
   416  
   417  		ginkgo.By("deleting")
   418  		err = csrClient.Delete(ctx, createdCSR.Name, metav1.DeleteOptions{})
   419  		framework.ExpectNoError(err)
   420  		_, err = csrClient.Get(ctx, createdCSR.Name, metav1.GetOptions{})
   421  		if !apierrors.IsNotFound(err) {
   422  			framework.Failf("expected 404, got %#v", err)
   423  		}
   424  		csrs, err = csrClient.List(ctx, metav1.ListOptions{FieldSelector: "spec.signerName=" + signerName})
   425  		framework.ExpectNoError(err)
   426  		gomega.Expect(csrs.Items).To(gomega.HaveLen(2), "filtered list should have 2 items")
   427  
   428  		ginkgo.By("deleting a collection")
   429  		err = csrClient.DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: "spec.signerName=" + signerName})
   430  		framework.ExpectNoError(err)
   431  		csrs, err = csrClient.List(ctx, metav1.ListOptions{FieldSelector: "spec.signerName=" + signerName})
   432  		framework.ExpectNoError(err)
   433  		gomega.Expect(csrs.Items).To(gomega.BeEmpty(), "filtered list should have 0 items")
   434  	})
   435  })
   436
View as plain text