xds_resolver.go

Documentation: google.golang.org/grpc/xds/internal/resolver

     1  /*
     2   * Copyright 2019 gRPC authors.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   *
    16   */
    17  
    18  // Package resolver implements the xds resolver, that does LDS and RDS to find
    19  // the cluster to use.
    20  package resolver
    21  
    22  import (
    23  	"context"
    24  	"fmt"
    25  	"sync/atomic"
    26  
    27  	"google.golang.org/grpc/internal"
    28  	"google.golang.org/grpc/internal/grpclog"
    29  	"google.golang.org/grpc/internal/grpcrand"
    30  	"google.golang.org/grpc/internal/grpcsync"
    31  	"google.golang.org/grpc/internal/pretty"
    32  	iresolver "google.golang.org/grpc/internal/resolver"
    33  	"google.golang.org/grpc/internal/wrr"
    34  	"google.golang.org/grpc/internal/xds/bootstrap"
    35  	"google.golang.org/grpc/resolver"
    36  	rinternal "google.golang.org/grpc/xds/internal/resolver/internal"
    37  	"google.golang.org/grpc/xds/internal/xdsclient"
    38  	"google.golang.org/grpc/xds/internal/xdsclient/xdsresource"
    39  )
    40  
    41  // Scheme is the xDS resolver's scheme.
    42  //
    43  // TODO(easwars): Rename this package as xdsresolver so that this is accessed as
    44  // xdsresolver.Scheme
    45  const Scheme = "xds"
    46  
    47  // newBuilderForTesting creates a new xds resolver builder using a specific xds
    48  // bootstrap config, so tests can use multiple xds clients in different
    49  // ClientConns at the same time.
    50  func newBuilderForTesting(config []byte) (resolver.Builder, error) {
    51  	return &xdsResolverBuilder{
    52  		newXDSClient: func() (xdsclient.XDSClient, func(), error) {
    53  			return xdsclient.NewWithBootstrapContentsForTesting(config)
    54  		},
    55  	}, nil
    56  }
    57  
    58  func init() {
    59  	resolver.Register(&xdsResolverBuilder{})
    60  	internal.NewXDSResolverWithConfigForTesting = newBuilderForTesting
    61  
    62  	rinternal.NewWRR = wrr.NewRandom
    63  	rinternal.NewXDSClient = xdsclient.New
    64  }
    65  
    66  type xdsResolverBuilder struct {
    67  	newXDSClient func() (xdsclient.XDSClient, func(), error)
    68  }
    69  
    70  // Build helps implement the resolver.Builder interface.
    71  //
    72  // The xds bootstrap process is performed (and a new xds client is built) every
    73  // time an xds resolver is built.
    74  func (b *xdsResolverBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (_ resolver.Resolver, retErr error) {
    75  	r := &xdsResolver{
    76  		cc:             cc,
    77  		activeClusters: make(map[string]*clusterInfo),
    78  		channelID:      grpcrand.Uint64(),
    79  	}
    80  	defer func() {
    81  		if retErr != nil {
    82  			r.Close()
    83  		}
    84  	}()
    85  	r.logger = prefixLogger(r)
    86  	r.logger.Infof("Creating resolver for target: %+v", target)
    87  
    88  	// Initialize the serializer used to synchronize the following:
    89  	// - updates from the xDS client. This could lead to generation of new
    90  	//   service config if resolution is complete.
    91  	// - completion of an RPC to a removed cluster causing the associated ref
    92  	//   count to become zero, resulting in generation of new service config.
    93  	// - stopping of a config selector that results in generation of new service
    94  	//   config.
    95  	ctx, cancel := context.WithCancel(context.Background())
    96  	r.serializer = grpcsync.NewCallbackSerializer(ctx)
    97  	r.serializerCancel = cancel
    98  
    99  	// Initialize the xDS client.
   100  	newXDSClient := rinternal.NewXDSClient.(func() (xdsclient.XDSClient, func(), error))
   101  	if b.newXDSClient != nil {
   102  		newXDSClient = b.newXDSClient
   103  	}
   104  	client, close, err := newXDSClient()
   105  	if err != nil {
   106  		return nil, fmt.Errorf("xds: failed to create xds-client: %v", err)
   107  	}
   108  	r.xdsClient = client
   109  	r.xdsClientClose = close
   110  
   111  	// Determine the listener resource name and start a watcher for it.
   112  	template, err := r.sanityChecksOnBootstrapConfig(target, opts, r.xdsClient)
   113  	if err != nil {
   114  		return nil, err
   115  	}
   116  	r.dataplaneAuthority = opts.Authority
   117  	r.ldsResourceName = bootstrap.PopulateResourceTemplate(template, target.Endpoint())
   118  	r.listenerWatcher = newListenerWatcher(r.ldsResourceName, r)
   119  	return r, nil
   120  }
   121  
   122  // Performs the following sanity checks:
   123  //   - Verifies that the bootstrap configuration is not empty.
   124  //   - Verifies that if xDS credentials are specified by the user, the
   125  //     bootstrap configuration contains certificate providers.
   126  //   - Verifies that if the provided dial target contains an authority, the
   127  //     bootstrap configuration contains server config for that authority.
   128  //
   129  // Returns the listener resource name template to use. If any of the above
   130  // validations fail, a non-nil error is returned.
   131  func (r *xdsResolver) sanityChecksOnBootstrapConfig(target resolver.Target, opts resolver.BuildOptions, client xdsclient.XDSClient) (string, error) {
   132  	bootstrapConfig := client.BootstrapConfig()
   133  	if bootstrapConfig == nil {
   134  		// This is never expected to happen after a successful xDS client
   135  		// creation. Defensive programming.
   136  		return "", fmt.Errorf("xds: bootstrap configuration is empty")
   137  	}
   138  
   139  	// Find the client listener template to use from the bootstrap config:
   140  	// - If authority is not set in the target, use the top level template
   141  	// - If authority is set, use the template from the authority map.
   142  	template := bootstrapConfig.ClientDefaultListenerResourceNameTemplate
   143  	if authority := target.URL.Host; authority != "" {
   144  		a := bootstrapConfig.Authorities[authority]
   145  		if a == nil {
   146  			return "", fmt.Errorf("xds: authority %q specified in dial target %q is not found in the bootstrap file", authority, target)
   147  		}
   148  		if a.ClientListenerResourceNameTemplate != "" {
   149  			// This check will never be false, because
   150  			// ClientListenerResourceNameTemplate is required to start with
   151  			// xdstp://, and has a default value (not an empty string) if unset.
   152  			template = a.ClientListenerResourceNameTemplate
   153  		}
   154  	}
   155  	return template, nil
   156  }
   157  
   158  // Name helps implement the resolver.Builder interface.
   159  func (*xdsResolverBuilder) Scheme() string {
   160  	return Scheme
   161  }
   162  
   163  // xdsResolver implements the resolver.Resolver interface.
   164  //
   165  // It registers a watcher for ServiceConfig updates with the xdsClient object
   166  // (which performs LDS/RDS queries for the same), and passes the received
   167  // updates to the ClientConn.
   168  type xdsResolver struct {
   169  	cc     resolver.ClientConn
   170  	logger *grpclog.PrefixLogger
   171  	// The underlying xdsClient which performs all xDS requests and responses.
   172  	xdsClient      xdsclient.XDSClient
   173  	xdsClientClose func()
   174  	// A random number which uniquely identifies the channel which owns this
   175  	// resolver.
   176  	channelID uint64
   177  
   178  	// All methods on the xdsResolver type except for the ones invoked by gRPC,
   179  	// i.e ResolveNow() and Close(), are guaranteed to execute in the context of
   180  	// this serializer's callback. And since the serializer guarantees mutual
   181  	// exclusion among these callbacks, we can get by without any mutexes to
   182  	// access all of the below defined state. The only exception is Close(),
   183  	// which does access some of this shared state, but it does so after
   184  	// cancelling the context passed to the serializer.
   185  	serializer       *grpcsync.CallbackSerializer
   186  	serializerCancel context.CancelFunc
   187  
   188  	// dataplaneAuthority is the authority used for the data plane connections,
   189  	// which is also used to select the VirtualHost within the xDS
   190  	// RouteConfiguration.  This is %-encoded to match with VirtualHost Domain
   191  	// in xDS RouteConfiguration.
   192  	dataplaneAuthority string
   193  
   194  	ldsResourceName     string
   195  	listenerWatcher     *listenerWatcher
   196  	listenerUpdateRecvd bool
   197  	currentListener     xdsresource.ListenerUpdate
   198  
   199  	rdsResourceName        string
   200  	routeConfigWatcher     *routeConfigWatcher
   201  	routeConfigUpdateRecvd bool
   202  	currentRouteConfig     xdsresource.RouteConfigUpdate
   203  	currentVirtualHost     *xdsresource.VirtualHost // Matched virtual host for quick access.
   204  
   205  	// activeClusters is a map from cluster name to information about the
   206  	// cluster that includes a ref count and load balancing configuration.
   207  	activeClusters map[string]*clusterInfo
   208  
   209  	curConfigSelector *configSelector
   210  }
   211  
   212  // ResolveNow is a no-op at this point.
   213  func (*xdsResolver) ResolveNow(o resolver.ResolveNowOptions) {}
   214  
   215  func (r *xdsResolver) Close() {
   216  	// Cancel the context passed to the serializer and wait for any scheduled
   217  	// callbacks to complete. Canceling the context ensures that no new
   218  	// callbacks will be scheduled.
   219  	r.serializerCancel()
   220  	<-r.serializer.Done()
   221  
   222  	// Note that Close needs to check for nils even if some of them are always
   223  	// set in the constructor. This is because the constructor defers Close() in
   224  	// error cases, and the fields might not be set when the error happens.
   225  
   226  	if r.listenerWatcher != nil {
   227  		r.listenerWatcher.stop()
   228  	}
   229  	if r.routeConfigWatcher != nil {
   230  		r.routeConfigWatcher.stop()
   231  	}
   232  	if r.xdsClientClose != nil {
   233  		r.xdsClientClose()
   234  	}
   235  	r.logger.Infof("Shutdown")
   236  }
   237  
   238  // sendNewServiceConfig prunes active clusters, generates a new service config
   239  // based on the current set of active clusters, and sends an update to the
   240  // channel with that service config and the provided config selector.  Returns
   241  // false if an error occurs while generating the service config and the update
   242  // cannot be sent.
   243  //
   244  // Only executed in the context of a serializer callback.
   245  func (r *xdsResolver) sendNewServiceConfig(cs *configSelector) bool {
   246  	// Delete entries from r.activeClusters with zero references;
   247  	// otherwise serviceConfigJSON will generate a config including
   248  	// them.
   249  	r.pruneActiveClusters()
   250  
   251  	if cs == nil && len(r.activeClusters) == 0 {
   252  		// There are no clusters and we are sending a failing configSelector.
   253  		// Send an empty config, which picks pick-first, with no address, and
   254  		// puts the ClientConn into transient failure.
   255  		r.cc.UpdateState(resolver.State{ServiceConfig: r.cc.ParseServiceConfig("{}")})
   256  		return true
   257  	}
   258  
   259  	sc, err := serviceConfigJSON(r.activeClusters)
   260  	if err != nil {
   261  		// JSON marshal error; should never happen.
   262  		r.logger.Errorf("For Listener resource %q and RouteConfiguration resource %q, failed to marshal newly built service config: %v", r.ldsResourceName, r.rdsResourceName, err)
   263  		r.cc.ReportError(err)
   264  		return false
   265  	}
   266  	r.logger.Infof("For Listener resource %q and RouteConfiguration resource %q, generated service config: %v", r.ldsResourceName, r.rdsResourceName, pretty.FormatJSON(sc))
   267  
   268  	// Send the update to the ClientConn.
   269  	state := iresolver.SetConfigSelector(resolver.State{
   270  		ServiceConfig: r.cc.ParseServiceConfig(string(sc)),
   271  	}, cs)
   272  	r.cc.UpdateState(xdsclient.SetClient(state, r.xdsClient))
   273  	return true
   274  }
   275  
   276  // newConfigSelector creates a new config selector using the most recently
   277  // received listener and route config updates. May add entries to
   278  // r.activeClusters for previously-unseen clusters.
   279  //
   280  // Only executed in the context of a serializer callback.
   281  func (r *xdsResolver) newConfigSelector() (*configSelector, error) {
   282  	cs := &configSelector{
   283  		r: r,
   284  		virtualHost: virtualHost{
   285  			httpFilterConfigOverride: r.currentVirtualHost.HTTPFilterConfigOverride,
   286  			retryConfig:              r.currentVirtualHost.RetryConfig,
   287  		},
   288  		routes:           make([]route, len(r.currentVirtualHost.Routes)),
   289  		clusters:         make(map[string]*clusterInfo),
   290  		httpFilterConfig: r.currentListener.HTTPFilters,
   291  	}
   292  
   293  	for i, rt := range r.currentVirtualHost.Routes {
   294  		clusters := rinternal.NewWRR.(func() wrr.WRR)()
   295  		if rt.ClusterSpecifierPlugin != "" {
   296  			clusterName := clusterSpecifierPluginPrefix + rt.ClusterSpecifierPlugin
   297  			clusters.Add(&routeCluster{
   298  				name: clusterName,
   299  			}, 1)
   300  			ci := r.addOrGetActiveClusterInfo(clusterName)
   301  			ci.cfg = xdsChildConfig{ChildPolicy: balancerConfig(r.currentRouteConfig.ClusterSpecifierPlugins[rt.ClusterSpecifierPlugin])}
   302  			cs.clusters[clusterName] = ci
   303  		} else {
   304  			for cluster, wc := range rt.WeightedClusters {
   305  				clusterName := clusterPrefix + cluster
   306  				clusters.Add(&routeCluster{
   307  					name:                     clusterName,
   308  					httpFilterConfigOverride: wc.HTTPFilterConfigOverride,
   309  				}, int64(wc.Weight))
   310  				ci := r.addOrGetActiveClusterInfo(clusterName)
   311  				ci.cfg = xdsChildConfig{ChildPolicy: newBalancerConfig(cdsName, cdsBalancerConfig{Cluster: cluster})}
   312  				cs.clusters[clusterName] = ci
   313  			}
   314  		}
   315  		cs.routes[i].clusters = clusters
   316  
   317  		var err error
   318  		cs.routes[i].m, err = xdsresource.RouteToMatcher(rt)
   319  		if err != nil {
   320  			return nil, err
   321  		}
   322  		cs.routes[i].actionType = rt.ActionType
   323  		if rt.MaxStreamDuration == nil {
   324  			cs.routes[i].maxStreamDuration = r.currentListener.MaxStreamDuration
   325  		} else {
   326  			cs.routes[i].maxStreamDuration = *rt.MaxStreamDuration
   327  		}
   328  
   329  		cs.routes[i].httpFilterConfigOverride = rt.HTTPFilterConfigOverride
   330  		cs.routes[i].retryConfig = rt.RetryConfig
   331  		cs.routes[i].hashPolicies = rt.HashPolicies
   332  	}
   333  
   334  	// Account for this config selector's clusters.  Do this after no further
   335  	// errors may occur.  Note: cs.clusters are pointers to entries in
   336  	// activeClusters.
   337  	for _, ci := range cs.clusters {
   338  		atomic.AddInt32(&ci.refCount, 1)
   339  	}
   340  
   341  	return cs, nil
   342  }
   343  
   344  // pruneActiveClusters deletes entries in r.activeClusters with zero
   345  // references.
   346  func (r *xdsResolver) pruneActiveClusters() {
   347  	for cluster, ci := range r.activeClusters {
   348  		if atomic.LoadInt32(&ci.refCount) == 0 {
   349  			delete(r.activeClusters, cluster)
   350  		}
   351  	}
   352  }
   353  
   354  func (r *xdsResolver) addOrGetActiveClusterInfo(name string) *clusterInfo {
   355  	ci := r.activeClusters[name]
   356  	if ci != nil {
   357  		return ci
   358  	}
   359  
   360  	ci = &clusterInfo{refCount: 0}
   361  	r.activeClusters[name] = ci
   362  	return ci
   363  }
   364  
   365  type clusterInfo struct {
   366  	// number of references to this cluster; accessed atomically
   367  	refCount int32
   368  	// cfg is the child configuration for this cluster, containing either the
   369  	// csp config or the cds cluster config.
   370  	cfg xdsChildConfig
   371  }
   372  
   373  // Determines if the xdsResolver has received all required configuration, i.e
   374  // Listener and RouteConfiguration resources, from the management server, and
   375  // whether a matching virtual host was found in the RouteConfiguration resource.
   376  func (r *xdsResolver) resolutionComplete() bool {
   377  	return r.listenerUpdateRecvd && r.routeConfigUpdateRecvd && r.currentVirtualHost != nil
   378  }
   379  
   380  // onResolutionComplete performs the following actions when resolution is
   381  // complete, i.e Listener and RouteConfiguration resources have been received
   382  // from the management server and a matching virtual host is found in the
   383  // latter.
   384  //   - creates a new config selector (this involves incrementing references to
   385  //     clusters owned by this config selector).
   386  //   - stops the old config selector (this involves decrementing references to
   387  //     clusters owned by this config selector).
   388  //   - prunes active clusters and pushes a new service config to the channel.
   389  //   - updates the current config selector used by the resolver.
   390  //
   391  // Only executed in the context of a serializer callback.
   392  func (r *xdsResolver) onResolutionComplete() {
   393  	if !r.resolutionComplete() {
   394  		return
   395  	}
   396  
   397  	cs, err := r.newConfigSelector()
   398  	if err != nil {
   399  		r.logger.Warningf("Failed to build a config selector for resource %q: %v", r.ldsResourceName, err)
   400  		r.cc.ReportError(err)
   401  		return
   402  	}
   403  
   404  	if !r.sendNewServiceConfig(cs) {
   405  		// JSON error creating the service config (unexpected); erase
   406  		// this config selector and ignore this update, continuing with
   407  		// the previous config selector.
   408  		cs.stop()
   409  		return
   410  	}
   411  
   412  	r.curConfigSelector.stop()
   413  	r.curConfigSelector = cs
   414  }
   415  
   416  func (r *xdsResolver) applyRouteConfigUpdate(update xdsresource.RouteConfigUpdate) {
   417  	matchVh := xdsresource.FindBestMatchingVirtualHost(r.dataplaneAuthority, update.VirtualHosts)
   418  	if matchVh == nil {
   419  		r.onError(fmt.Errorf("no matching virtual host found for %q", r.dataplaneAuthority))
   420  		return
   421  	}
   422  	r.currentRouteConfig = update
   423  	r.currentVirtualHost = matchVh
   424  	r.routeConfigUpdateRecvd = true
   425  
   426  	r.onResolutionComplete()
   427  }
   428  
   429  // onError propagates the error up to the channel. And since this is invoked
   430  // only for non resource-not-found errors, we don't have to update resolver
   431  // state and we can keep using the old config.
   432  //
   433  // Only executed in the context of a serializer callback.
   434  func (r *xdsResolver) onError(err error) {
   435  	r.cc.ReportError(err)
   436  }
   437  
   438  // Contains common functionality to be executed when resources of either type
   439  // are removed.
   440  //
   441  // Only executed in the context of a serializer callback.
   442  func (r *xdsResolver) onResourceNotFound() {
   443  	// We cannot remove clusters from the service config that have ongoing RPCs.
   444  	// Instead, what we can do is to send an erroring (nil) config selector
   445  	// along with normal service config. This will ensure that new RPCs will
   446  	// fail, and once the active RPCs complete, the reference counts on the
   447  	// clusters will come down to zero. At that point, we will send an empty
   448  	// service config with no addresses. This results in the pick-first
   449  	// LB policy being configured on the channel, and since there are no
   450  	// address, pick-first will put the channel in TRANSIENT_FAILURE.
   451  	r.sendNewServiceConfig(nil)
   452  
   453  	// Stop and dereference the active config selector, if one exists.
   454  	r.curConfigSelector.stop()
   455  	r.curConfigSelector = nil
   456  }
   457  
   458  // Only executed in the context of a serializer callback.
   459  func (r *xdsResolver) onListenerResourceUpdate(update xdsresource.ListenerUpdate) {
   460  	if r.logger.V(2) {
   461  		r.logger.Infof("Received update for Listener resource %q: %v", r.ldsResourceName, pretty.ToJSON(update))
   462  	}
   463  
   464  	r.currentListener = update
   465  	r.listenerUpdateRecvd = true
   466  
   467  	if update.InlineRouteConfig != nil {
   468  		// If there was a previous route config watcher because of a non-inline
   469  		// route configuration, cancel it.
   470  		r.rdsResourceName = ""
   471  		if r.routeConfigWatcher != nil {
   472  			r.routeConfigWatcher.stop()
   473  			r.routeConfigWatcher = nil
   474  		}
   475  
   476  		r.applyRouteConfigUpdate(*update.InlineRouteConfig)
   477  		return
   478  	}
   479  
   480  	// We get here only if there was no inline route configuration.
   481  
   482  	// If the route config name has not changed, send an update with existing
   483  	// route configuration and the newly received listener configuration.
   484  	if r.rdsResourceName == update.RouteConfigName {
   485  		r.onResolutionComplete()
   486  		return
   487  	}
   488  
   489  	// If the route config name has changed, cancel the old watcher and start a
   490  	// new one. At this point, since we have not yet resolved the new route
   491  	// config name, we don't send an update to the channel, and therefore
   492  	// continue using the old route configuration (if received) until the new
   493  	// one is received.
   494  	r.rdsResourceName = update.RouteConfigName
   495  	if r.routeConfigWatcher != nil {
   496  		r.routeConfigWatcher.stop()
   497  		r.currentVirtualHost = nil
   498  		r.routeConfigUpdateRecvd = false
   499  	}
   500  	r.routeConfigWatcher = newRouteConfigWatcher(r.rdsResourceName, r)
   501  }
   502  
   503  func (r *xdsResolver) onListenerResourceError(err error) {
   504  	if r.logger.V(2) {
   505  		r.logger.Infof("Received error for Listener resource %q: %v", r.ldsResourceName, err)
   506  	}
   507  	r.onError(err)
   508  }
   509  
   510  // Only executed in the context of a serializer callback.
   511  func (r *xdsResolver) onListenerResourceNotFound() {
   512  	if r.logger.V(2) {
   513  		r.logger.Infof("Received resource-not-found-error for Listener resource %q", r.ldsResourceName)
   514  	}
   515  
   516  	r.listenerUpdateRecvd = false
   517  
   518  	if r.routeConfigWatcher != nil {
   519  		r.routeConfigWatcher.stop()
   520  	}
   521  	r.rdsResourceName = ""
   522  	r.currentVirtualHost = nil
   523  	r.routeConfigUpdateRecvd = false
   524  	r.routeConfigWatcher = nil
   525  
   526  	r.onResourceNotFound()
   527  }
   528  
   529  // Only executed in the context of a serializer callback.
   530  func (r *xdsResolver) onRouteConfigResourceUpdate(name string, update xdsresource.RouteConfigUpdate) {
   531  	if r.logger.V(2) {
   532  		r.logger.Infof("Received update for RouteConfiguration resource %q: %v", name, pretty.ToJSON(update))
   533  	}
   534  
   535  	if r.rdsResourceName != name {
   536  		// Drop updates from canceled watchers.
   537  		return
   538  	}
   539  
   540  	r.applyRouteConfigUpdate(update)
   541  }
   542  
   543  // Only executed in the context of a serializer callback.
   544  func (r *xdsResolver) onRouteConfigResourceError(name string, err error) {
   545  	if r.logger.V(2) {
   546  		r.logger.Infof("Received error for RouteConfiguration resource %q: %v", name, err)
   547  	}
   548  	r.onError(err)
   549  }
   550  
   551  // Only executed in the context of a serializer callback.
   552  func (r *xdsResolver) onRouteConfigResourceNotFound(name string) {
   553  	if r.logger.V(2) {
   554  		r.logger.Infof("Received resource-not-found-error for RouteConfiguration resource %q", name)
   555  	}
   556  
   557  	if r.rdsResourceName != name {
   558  		return
   559  	}
   560  	r.onResourceNotFound()
   561  }
   562  
   563  // Only executed in the context of a serializer callback.
   564  func (r *xdsResolver) onClusterRefDownToZero() {
   565  	r.sendNewServiceConfig(r.curConfigSelector)
   566  }
   567
View as plain text