storage_rbac.go

Documentation: k8s.io/kubernetes/pkg/registry/rbac/rest

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package rest
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"time"
    23  
    24  	"k8s.io/klog/v2"
    25  
    26  	rbacapiv1 "k8s.io/api/rbac/v1"
    27  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    28  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    29  	"k8s.io/apimachinery/pkg/runtime/schema"
    30  	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
    31  	"k8s.io/apimachinery/pkg/util/wait"
    32  	"k8s.io/apiserver/pkg/authorization/authorizer"
    33  	"k8s.io/apiserver/pkg/registry/generic"
    34  	"k8s.io/apiserver/pkg/registry/rest"
    35  	genericapiserver "k8s.io/apiserver/pkg/server"
    36  	serverstorage "k8s.io/apiserver/pkg/server/storage"
    37  	clientset "k8s.io/client-go/kubernetes"
    38  	rbacv1client "k8s.io/client-go/kubernetes/typed/rbac/v1"
    39  	"k8s.io/client-go/util/retry"
    40  	"k8s.io/component-helpers/auth/rbac/reconciliation"
    41  	"k8s.io/kubernetes/pkg/api/legacyscheme"
    42  	"k8s.io/kubernetes/pkg/apis/rbac"
    43  	"k8s.io/kubernetes/pkg/registry/rbac/clusterrole"
    44  	clusterrolepolicybased "k8s.io/kubernetes/pkg/registry/rbac/clusterrole/policybased"
    45  	clusterrolestore "k8s.io/kubernetes/pkg/registry/rbac/clusterrole/storage"
    46  	"k8s.io/kubernetes/pkg/registry/rbac/clusterrolebinding"
    47  	clusterrolebindingpolicybased "k8s.io/kubernetes/pkg/registry/rbac/clusterrolebinding/policybased"
    48  	clusterrolebindingstore "k8s.io/kubernetes/pkg/registry/rbac/clusterrolebinding/storage"
    49  	"k8s.io/kubernetes/pkg/registry/rbac/role"
    50  	rolepolicybased "k8s.io/kubernetes/pkg/registry/rbac/role/policybased"
    51  	rolestore "k8s.io/kubernetes/pkg/registry/rbac/role/storage"
    52  	"k8s.io/kubernetes/pkg/registry/rbac/rolebinding"
    53  	rolebindingpolicybased "k8s.io/kubernetes/pkg/registry/rbac/rolebinding/policybased"
    54  	rolebindingstore "k8s.io/kubernetes/pkg/registry/rbac/rolebinding/storage"
    55  	rbacregistryvalidation "k8s.io/kubernetes/pkg/registry/rbac/validation"
    56  	"k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac/bootstrappolicy"
    57  )
    58  
    59  const PostStartHookName = "rbac/bootstrap-roles"
    60  
    61  type RESTStorageProvider struct {
    62  	Authorizer authorizer.Authorizer
    63  }
    64  
    65  var _ genericapiserver.PostStartHookProvider = RESTStorageProvider{}
    66  
    67  func (p RESTStorageProvider) NewRESTStorage(apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (genericapiserver.APIGroupInfo, error) {
    68  	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(rbac.GroupName, legacyscheme.Scheme, legacyscheme.ParameterCodec, legacyscheme.Codecs)
    69  	// If you add a version here, be sure to add an entry in `k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go with specific priorities.
    70  	// TODO refactor the plumbing to provide the information in the APIGroupInfo
    71  
    72  	if storageMap, err := p.storage(rbacapiv1.SchemeGroupVersion, apiResourceConfigSource, restOptionsGetter); err != nil {
    73  		return genericapiserver.APIGroupInfo{}, err
    74  	} else if len(storageMap) > 0 {
    75  		apiGroupInfo.VersionedResourcesStorageMap[rbacapiv1.SchemeGroupVersion.Version] = storageMap
    76  	}
    77  
    78  	return apiGroupInfo, nil
    79  }
    80  
    81  func (p RESTStorageProvider) storage(version schema.GroupVersion, apiResourceConfigSource serverstorage.APIResourceConfigSource, restOptionsGetter generic.RESTOptionsGetter) (map[string]rest.Storage, error) {
    82  	storage := map[string]rest.Storage{}
    83  
    84  	rolesStorage, err := rolestore.NewREST(restOptionsGetter)
    85  	if err != nil {
    86  		return storage, err
    87  	}
    88  	roleBindingsStorage, err := rolebindingstore.NewREST(restOptionsGetter)
    89  	if err != nil {
    90  		return storage, err
    91  	}
    92  	clusterRolesStorage, err := clusterrolestore.NewREST(restOptionsGetter)
    93  	if err != nil {
    94  		return storage, err
    95  	}
    96  	clusterRoleBindingsStorage, err := clusterrolebindingstore.NewREST(restOptionsGetter)
    97  	if err != nil {
    98  		return storage, err
    99  	}
   100  
   101  	authorizationRuleResolver := rbacregistryvalidation.NewDefaultRuleResolver(
   102  		role.AuthorizerAdapter{Registry: role.NewRegistry(rolesStorage)},
   103  		rolebinding.AuthorizerAdapter{Registry: rolebinding.NewRegistry(roleBindingsStorage)},
   104  		clusterrole.AuthorizerAdapter{Registry: clusterrole.NewRegistry(clusterRolesStorage)},
   105  		clusterrolebinding.AuthorizerAdapter{Registry: clusterrolebinding.NewRegistry(clusterRoleBindingsStorage)},
   106  	)
   107  
   108  	// roles
   109  	if resource := "roles"; apiResourceConfigSource.ResourceEnabled(rbacapiv1.SchemeGroupVersion.WithResource(resource)) {
   110  		storage[resource] = rolepolicybased.NewStorage(rolesStorage, p.Authorizer, authorizationRuleResolver)
   111  	}
   112  
   113  	// rolebindings
   114  	if resource := "rolebindings"; apiResourceConfigSource.ResourceEnabled(rbacapiv1.SchemeGroupVersion.WithResource(resource)) {
   115  		storage[resource] = rolebindingpolicybased.NewStorage(roleBindingsStorage, p.Authorizer, authorizationRuleResolver)
   116  	}
   117  
   118  	// clusterroles
   119  	if resource := "clusterroles"; apiResourceConfigSource.ResourceEnabled(rbacapiv1.SchemeGroupVersion.WithResource(resource)) {
   120  		storage[resource] = clusterrolepolicybased.NewStorage(clusterRolesStorage, p.Authorizer, authorizationRuleResolver)
   121  	}
   122  
   123  	// clusterrolebindings
   124  	if resource := "clusterrolebindings"; apiResourceConfigSource.ResourceEnabled(rbacapiv1.SchemeGroupVersion.WithResource(resource)) {
   125  		storage[resource] = clusterrolebindingpolicybased.NewStorage(clusterRoleBindingsStorage, p.Authorizer, authorizationRuleResolver)
   126  	}
   127  
   128  	return storage, nil
   129  }
   130  
   131  func (p RESTStorageProvider) PostStartHook() (string, genericapiserver.PostStartHookFunc, error) {
   132  	policy := &PolicyData{
   133  		ClusterRoles:               append(bootstrappolicy.ClusterRoles(), bootstrappolicy.ControllerRoles()...),
   134  		ClusterRoleBindings:        append(bootstrappolicy.ClusterRoleBindings(), bootstrappolicy.ControllerRoleBindings()...),
   135  		Roles:                      bootstrappolicy.NamespaceRoles(),
   136  		RoleBindings:               bootstrappolicy.NamespaceRoleBindings(),
   137  		ClusterRolesToAggregate:    bootstrappolicy.ClusterRolesToAggregate(),
   138  		ClusterRoleBindingsToSplit: bootstrappolicy.ClusterRoleBindingsToSplit(),
   139  	}
   140  	return PostStartHookName, policy.EnsureRBACPolicy(), nil
   141  }
   142  
   143  type PolicyData struct {
   144  	ClusterRoles        []rbacapiv1.ClusterRole
   145  	ClusterRoleBindings []rbacapiv1.ClusterRoleBinding
   146  	Roles               map[string][]rbacapiv1.Role
   147  	RoleBindings        map[string][]rbacapiv1.RoleBinding
   148  	// ClusterRolesToAggregate maps from previous clusterrole name to the new clusterrole name
   149  	ClusterRolesToAggregate map[string]string
   150  	// ClusterRoleBindingsToSplit maps from previous ClusterRoleBinding Name to a template for the new ClusterRoleBinding
   151  	ClusterRoleBindingsToSplit map[string]rbacapiv1.ClusterRoleBinding
   152  }
   153  
   154  func isConflictOrServiceUnavailable(err error) bool {
   155  	return apierrors.IsConflict(err) || apierrors.IsServiceUnavailable(err)
   156  }
   157  
   158  func retryOnConflictOrServiceUnavailable(backoff wait.Backoff, fn func() error) error {
   159  	return retry.OnError(backoff, isConflictOrServiceUnavailable, fn)
   160  }
   161  
   162  func (p *PolicyData) EnsureRBACPolicy() genericapiserver.PostStartHookFunc {
   163  	return func(hookContext genericapiserver.PostStartHookContext) error {
   164  		// initializing roles is really important.  On some e2e runs, we've seen cases where etcd is down when the server
   165  		// starts, the roles don't initialize, and nothing works.
   166  		err := wait.Poll(1*time.Second, 30*time.Second, func() (done bool, err error) {
   167  			client, err := clientset.NewForConfig(hookContext.LoopbackClientConfig)
   168  			if err != nil {
   169  				utilruntime.HandleError(fmt.Errorf("unable to initialize client set: %v", err))
   170  				return false, nil
   171  			}
   172  			return ensureRBACPolicy(p, client)
   173  		})
   174  		// if we're never able to make it through initialization, kill the API server
   175  		if err != nil {
   176  			return fmt.Errorf("unable to initialize roles: %v", err)
   177  		}
   178  
   179  		return nil
   180  	}
   181  }
   182  
   183  func ensureRBACPolicy(p *PolicyData, client clientset.Interface) (done bool, err error) {
   184  	failedReconciliation := false
   185  	// Make sure etcd is responding before we start reconciling
   186  	if _, err := client.RbacV1().ClusterRoles().List(context.TODO(), metav1.ListOptions{}); err != nil {
   187  		utilruntime.HandleError(fmt.Errorf("unable to initialize clusterroles: %v", err))
   188  		return false, nil
   189  	}
   190  	if _, err := client.RbacV1().ClusterRoleBindings().List(context.TODO(), metav1.ListOptions{}); err != nil {
   191  		utilruntime.HandleError(fmt.Errorf("unable to initialize clusterrolebindings: %v", err))
   192  		return false, nil
   193  	}
   194  
   195  	// if the new cluster roles to aggregate do not yet exist, then we need to copy the old roles if they don't exist
   196  	// in new locations
   197  	if err := primeAggregatedClusterRoles(p.ClusterRolesToAggregate, client.RbacV1()); err != nil {
   198  		utilruntime.HandleError(fmt.Errorf("unable to prime aggregated clusterroles: %v", err))
   199  		return false, nil
   200  	}
   201  
   202  	if err := primeSplitClusterRoleBindings(p.ClusterRoleBindingsToSplit, client.RbacV1()); err != nil {
   203  		utilruntime.HandleError(fmt.Errorf("unable to prime split ClusterRoleBindings: %v", err))
   204  		return false, nil
   205  	}
   206  
   207  	// ensure bootstrap roles are created or reconciled
   208  	for _, clusterRole := range p.ClusterRoles {
   209  		opts := reconciliation.ReconcileRoleOptions{
   210  			Role:    reconciliation.ClusterRoleRuleOwner{ClusterRole: &clusterRole},
   211  			Client:  reconciliation.ClusterRoleModifier{Client: client.RbacV1().ClusterRoles()},
   212  			Confirm: true,
   213  		}
   214  		// ServiceUnavailble error is returned when the API server is blocked by storage version updates
   215  		err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
   216  			result, err := opts.Run()
   217  			if err != nil {
   218  				return err
   219  			}
   220  			switch {
   221  			case result.Protected && result.Operation != reconciliation.ReconcileNone:
   222  				klog.Warningf("skipped reconcile-protected clusterrole.%s/%s with missing permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules)
   223  			case result.Operation == reconciliation.ReconcileUpdate:
   224  				klog.V(2).Infof("updated clusterrole.%s/%s with additional permissions: %v", rbac.GroupName, clusterRole.Name, result.MissingRules)
   225  			case result.Operation == reconciliation.ReconcileCreate:
   226  				klog.V(2).Infof("created clusterrole.%s/%s", rbac.GroupName, clusterRole.Name)
   227  			}
   228  			return nil
   229  		})
   230  		if err != nil {
   231  			// don't fail on failures, try to create as many as you can
   232  			utilruntime.HandleError(fmt.Errorf("unable to reconcile clusterrole.%s/%s: %v", rbac.GroupName, clusterRole.Name, err))
   233  			failedReconciliation = true
   234  		}
   235  	}
   236  
   237  	// ensure bootstrap rolebindings are created or reconciled
   238  	for _, clusterRoleBinding := range p.ClusterRoleBindings {
   239  		opts := reconciliation.ReconcileRoleBindingOptions{
   240  			RoleBinding: reconciliation.ClusterRoleBindingAdapter{ClusterRoleBinding: &clusterRoleBinding},
   241  			Client:      reconciliation.ClusterRoleBindingClientAdapter{Client: client.RbacV1().ClusterRoleBindings()},
   242  			Confirm:     true,
   243  		}
   244  		// ServiceUnavailble error is returned when the API server is blocked by storage version updates
   245  		err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
   246  			result, err := opts.Run()
   247  			if err != nil {
   248  				return err
   249  			}
   250  			switch {
   251  			case result.Protected && result.Operation != reconciliation.ReconcileNone:
   252  				klog.Warningf("skipped reconcile-protected clusterrolebinding.%s/%s with missing subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects)
   253  			case result.Operation == reconciliation.ReconcileUpdate:
   254  				klog.V(2).Infof("updated clusterrolebinding.%s/%s with additional subjects: %v", rbac.GroupName, clusterRoleBinding.Name, result.MissingSubjects)
   255  			case result.Operation == reconciliation.ReconcileCreate:
   256  				klog.V(2).Infof("created clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name)
   257  			case result.Operation == reconciliation.ReconcileRecreate:
   258  				klog.V(2).Infof("recreated clusterrolebinding.%s/%s", rbac.GroupName, clusterRoleBinding.Name)
   259  			}
   260  			return nil
   261  		})
   262  		if err != nil {
   263  			// don't fail on failures, try to create as many as you can
   264  			utilruntime.HandleError(fmt.Errorf("unable to reconcile clusterrolebinding.%s/%s: %v", rbac.GroupName, clusterRoleBinding.Name, err))
   265  			failedReconciliation = true
   266  		}
   267  	}
   268  
   269  	// ensure bootstrap namespaced roles are created or reconciled
   270  	for namespace, roles := range p.Roles {
   271  		for _, role := range roles {
   272  			opts := reconciliation.ReconcileRoleOptions{
   273  				Role:    reconciliation.RoleRuleOwner{Role: &role},
   274  				Client:  reconciliation.RoleModifier{Client: client.RbacV1(), NamespaceClient: client.CoreV1().Namespaces()},
   275  				Confirm: true,
   276  			}
   277  			// ServiceUnavailble error is returned when the API server is blocked by storage version updates
   278  			err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
   279  				result, err := opts.Run()
   280  				if err != nil {
   281  					return err
   282  				}
   283  				switch {
   284  				case result.Protected && result.Operation != reconciliation.ReconcileNone:
   285  					klog.Warningf("skipped reconcile-protected role.%s/%s in %v with missing permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules)
   286  				case result.Operation == reconciliation.ReconcileUpdate:
   287  					klog.V(2).Infof("updated role.%s/%s in %v with additional permissions: %v", rbac.GroupName, role.Name, namespace, result.MissingRules)
   288  				case result.Operation == reconciliation.ReconcileCreate:
   289  					klog.V(2).Infof("created role.%s/%s in %v", rbac.GroupName, role.Name, namespace)
   290  				}
   291  				return nil
   292  			})
   293  			if err != nil {
   294  				// don't fail on failures, try to create as many as you can
   295  				utilruntime.HandleError(fmt.Errorf("unable to reconcile role.%s/%s in %v: %v", rbac.GroupName, role.Name, namespace, err))
   296  				failedReconciliation = true
   297  			}
   298  		}
   299  	}
   300  
   301  	// ensure bootstrap namespaced rolebindings are created or reconciled
   302  	for namespace, roleBindings := range p.RoleBindings {
   303  		for _, roleBinding := range roleBindings {
   304  			opts := reconciliation.ReconcileRoleBindingOptions{
   305  				RoleBinding: reconciliation.RoleBindingAdapter{RoleBinding: &roleBinding},
   306  				Client:      reconciliation.RoleBindingClientAdapter{Client: client.RbacV1(), NamespaceClient: client.CoreV1().Namespaces()},
   307  				Confirm:     true,
   308  			}
   309  			// ServiceUnavailble error is returned when the API server is blocked by storage version updates
   310  			err := retryOnConflictOrServiceUnavailable(retry.DefaultBackoff, func() error {
   311  				result, err := opts.Run()
   312  				if err != nil {
   313  					return err
   314  				}
   315  				switch {
   316  				case result.Protected && result.Operation != reconciliation.ReconcileNone:
   317  					klog.Warningf("skipped reconcile-protected rolebinding.%s/%s in %v with missing subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects)
   318  				case result.Operation == reconciliation.ReconcileUpdate:
   319  					klog.V(2).Infof("updated rolebinding.%s/%s in %v with additional subjects: %v", rbac.GroupName, roleBinding.Name, namespace, result.MissingSubjects)
   320  				case result.Operation == reconciliation.ReconcileCreate:
   321  					klog.V(2).Infof("created rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace)
   322  				case result.Operation == reconciliation.ReconcileRecreate:
   323  					klog.V(2).Infof("recreated rolebinding.%s/%s in %v", rbac.GroupName, roleBinding.Name, namespace)
   324  				}
   325  				return nil
   326  			})
   327  			if err != nil {
   328  				// don't fail on failures, try to create as many as you can
   329  				utilruntime.HandleError(fmt.Errorf("unable to reconcile rolebinding.%s/%s in %v: %v", rbac.GroupName, roleBinding.Name, namespace, err))
   330  				failedReconciliation = true
   331  			}
   332  		}
   333  	}
   334  	// failed to reconcile some objects, retry
   335  	if failedReconciliation {
   336  		return false, nil
   337  	}
   338  
   339  	return true, nil
   340  }
   341  
   342  func (p RESTStorageProvider) GroupName() string {
   343  	return rbac.GroupName
   344  }
   345  
   346  // primeAggregatedClusterRoles copies roles that have transitioned to aggregated roles and may need to pick up changes
   347  // that were done to the legacy roles.
   348  func primeAggregatedClusterRoles(clusterRolesToAggregate map[string]string, clusterRoleClient rbacv1client.ClusterRolesGetter) error {
   349  	for oldName, newName := range clusterRolesToAggregate {
   350  		_, err := clusterRoleClient.ClusterRoles().Get(context.TODO(), newName, metav1.GetOptions{})
   351  		if err == nil {
   352  			continue
   353  		}
   354  		if !apierrors.IsNotFound(err) {
   355  			return err
   356  		}
   357  
   358  		existingRole, err := clusterRoleClient.ClusterRoles().Get(context.TODO(), oldName, metav1.GetOptions{})
   359  		if apierrors.IsNotFound(err) {
   360  			continue
   361  		}
   362  		if err != nil {
   363  			return err
   364  		}
   365  		if existingRole.AggregationRule != nil {
   366  			// the old role already moved to an aggregated role, so there are no custom rules to migrate at this point
   367  			return nil
   368  		}
   369  		klog.V(1).Infof("migrating %v to %v", existingRole.Name, newName)
   370  		existingRole.Name = newName
   371  		existingRole.ResourceVersion = "" // clear this so the object can be created.
   372  		if _, err := clusterRoleClient.ClusterRoles().Create(context.TODO(), existingRole, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
   373  			return err
   374  		}
   375  	}
   376  
   377  	return nil
   378  }
   379  
   380  // primeSplitClusterRoleBindings ensures the existence of target ClusterRoleBindings
   381  // by copying Subjects, Annotations, and Labels from the specified source
   382  // ClusterRoleBinding, if present.
   383  func primeSplitClusterRoleBindings(clusterRoleBindingToSplit map[string]rbacapiv1.ClusterRoleBinding, clusterRoleBindingClient rbacv1client.ClusterRoleBindingsGetter) error {
   384  	for existingBindingName, clusterRoleBindingToCreate := range clusterRoleBindingToSplit {
   385  		// If source ClusterRoleBinding does not exist, do nothing.
   386  		existingRoleBinding, err := clusterRoleBindingClient.ClusterRoleBindings().Get(context.TODO(), existingBindingName, metav1.GetOptions{})
   387  		if apierrors.IsNotFound(err) {
   388  			continue
   389  		}
   390  		if err != nil {
   391  			return err
   392  		}
   393  
   394  		// If the target ClusterRoleBinding already exists, do nothing.
   395  		_, err = clusterRoleBindingClient.ClusterRoleBindings().Get(context.TODO(), clusterRoleBindingToCreate.Name, metav1.GetOptions{})
   396  		if err == nil {
   397  			continue
   398  		}
   399  		if !apierrors.IsNotFound(err) {
   400  			return err
   401  		}
   402  
   403  		// If the source exists, but the target does not,
   404  		// copy the subjects, labels, and annotations from the former to create the latter.
   405  		klog.V(1).Infof("copying subjects, labels, and annotations from ClusterRoleBinding %q to template %q", existingBindingName, clusterRoleBindingToCreate.Name)
   406  		newCRB := clusterRoleBindingToCreate.DeepCopy()
   407  		newCRB.Subjects = existingRoleBinding.Subjects
   408  		newCRB.Labels = existingRoleBinding.Labels
   409  		newCRB.Annotations = existingRoleBinding.Annotations
   410  		if _, err := clusterRoleBindingClient.ClusterRoleBindings().Create(context.TODO(), newCRB, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
   411  			return err
   412  		}
   413  	}
   414  	return nil
   415  }
   416
View as plain text