Source file src/google.golang.org/grpc/xds/internal/balancer/priority/balancer_test.go

Documentation: google.golang.org/grpc/xds/internal/balancer/priority

     1  /*
     2   *
     3   * Copyright 2021 gRPC authors.
     4   *
     5   * Licensed under the Apache License, Version 2.0 (the "License");
     6   * you may not use this file except in compliance with the License.
     7   * You may obtain a copy of the License at
     8   *
     9   *     http://www.apache.org/licenses/LICENSE-2.0
    10   *
    11   * Unless required by applicable law or agreed to in writing, software
    12   * distributed under the License is distributed on an "AS IS" BASIS,
    13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14   * See the License for the specific language governing permissions and
    15   * limitations under the License.
    16   *
    17   */
    18  
    19  package priority
    20  
    21  import (
    22  	"context"
    23  	"fmt"
    24  	"testing"
    25  	"time"
    26  
    27  	"google.golang.org/grpc/balancer"
    28  	"google.golang.org/grpc/balancer/roundrobin"
    29  	"google.golang.org/grpc/connectivity"
    30  	"google.golang.org/grpc/internal/balancer/stub"
    31  	"google.golang.org/grpc/internal/grpctest"
    32  	"google.golang.org/grpc/internal/hierarchy"
    33  	internalserviceconfig "google.golang.org/grpc/internal/serviceconfig"
    34  	"google.golang.org/grpc/internal/testutils"
    35  	"google.golang.org/grpc/resolver"
    36  )
    37  
    38  const (
    39  	defaultTestTimeout      = 5 * time.Second
    40  	defaultTestShortTimeout = 100 * time.Millisecond
    41  )
    42  
    43  type s struct {
    44  	grpctest.Tester
    45  }
    46  
    47  func Test(t *testing.T) {
    48  	grpctest.RunSubTests(t, s{})
    49  }
    50  
    51  var testBackendAddrStrs []string
    52  
    53  const (
    54  	testBackendAddrsCount = 12
    55  	testRRBalancerName    = "another-round-robin"
    56  )
    57  
    58  type anotherRR struct {
    59  	balancer.Builder
    60  }
    61  
    62  func (*anotherRR) Name() string {
    63  	return testRRBalancerName
    64  }
    65  
    66  func init() {
    67  	for i := 0; i < testBackendAddrsCount; i++ {
    68  		testBackendAddrStrs = append(testBackendAddrStrs, fmt.Sprintf("%d.%d.%d.%d:%d", i, i, i, i, i))
    69  	}
    70  	// Disable sub-balancer caching for all but the tests which exercise the
    71  	// caching behavior.
    72  	DefaultSubBalancerCloseTimeout = time.Duration(0)
    73  	balancer.Register(&anotherRR{Builder: balancer.Get(roundrobin.Name)})
    74  }
    75  
    76  // When a high priority is ready, adding/removing lower locality doesn't cause
    77  // changes.
    78  //
    79  // Init 0 and 1; 0 is up, use 0; add 2, use 0; remove 2, use 0.
    80  func (s) TestPriority_HighPriorityReady(t *testing.T) {
    81  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
    82  	defer cancel()
    83  
    84  	cc := testutils.NewBalancerClientConn(t)
    85  	bb := balancer.Get(Name)
    86  	pb := bb.Build(cc, balancer.BuildOptions{})
    87  	defer pb.Close()
    88  
    89  	// Two children, with priorities [0, 1], each with one backend.
    90  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
    91  		ResolverState: resolver.State{
    92  			Addresses: []resolver.Address{
    93  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
    94  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
    95  			},
    96  		},
    97  		BalancerConfig: &LBConfig{
    98  			Children: map[string]*Child{
    99  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   100  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   101  			},
   102  			Priorities: []string{"child-0", "child-1"},
   103  		},
   104  	}); err != nil {
   105  		t.Fatalf("failed to update ClientConn state: %v", err)
   106  	}
   107  
   108  	addrs1 := <-cc.NewSubConnAddrsCh
   109  	if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want {
   110  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   111  	}
   112  	sc1 := <-cc.NewSubConnCh
   113  
   114  	// p0 is ready.
   115  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   116  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   117  
   118  	// Test roundrobin with only p0 subconns.
   119  	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
   120  		t.Fatal(err.Error())
   121  	}
   122  
   123  	// Add p2, it shouldn't cause any updates.
   124  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   125  		ResolverState: resolver.State{
   126  			Addresses: []resolver.Address{
   127  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
   128  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
   129  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-2"}),
   130  			},
   131  		},
   132  		BalancerConfig: &LBConfig{
   133  			Children: map[string]*Child{
   134  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   135  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   136  				"child-2": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   137  			},
   138  			Priorities: []string{"child-0", "child-1", "child-2"},
   139  		},
   140  	}); err != nil {
   141  		t.Fatalf("failed to update ClientConn state: %v", err)
   142  	}
   143  
   144  	select {
   145  	case sc := <-cc.NewSubConnCh:
   146  		t.Fatalf("got unexpected new SubConn: %s", sc)
   147  	case sc := <-cc.ShutdownSubConnCh:
   148  		t.Fatalf("got unexpected shutdown SubConn: %v", sc)
   149  	case <-time.After(time.Millisecond * 100):
   150  	}
   151  
   152  	// Test roundrobin with only p0 subconns.
   153  	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
   154  		t.Fatal(err.Error())
   155  	}
   156  
   157  	// Remove p2, no updates.
   158  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   159  		ResolverState: resolver.State{
   160  			Addresses: []resolver.Address{
   161  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
   162  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
   163  			},
   164  		},
   165  		BalancerConfig: &LBConfig{
   166  			Children: map[string]*Child{
   167  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   168  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   169  			},
   170  			Priorities: []string{"child-0", "child-1"},
   171  		},
   172  	}); err != nil {
   173  		t.Fatalf("failed to update ClientConn state: %v", err)
   174  	}
   175  
   176  	select {
   177  	case <-cc.NewSubConnCh:
   178  		t.Fatalf("got unexpected new SubConn")
   179  	case <-cc.ShutdownSubConnCh:
   180  		t.Fatalf("got unexpected shutdown SubConn")
   181  	case <-time.After(time.Millisecond * 100):
   182  	}
   183  
   184  	// Test roundrobin with only p0 subconns.
   185  	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
   186  		t.Fatal(err.Error())
   187  	}
   188  }
   189  
   190  // Lower priority is used when higher priority is not ready.
   191  //
   192  // Init 0 and 1; 0 is up, use 0; 0 is down, 1 is up, use 1; add 2, use 1; 1 is
   193  // down, use 2; remove 2, use 1.
   194  func (s) TestPriority_SwitchPriority(t *testing.T) {
   195  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
   196  	defer cancel()
   197  
   198  	cc := testutils.NewBalancerClientConn(t)
   199  	bb := balancer.Get(Name)
   200  	pb := bb.Build(cc, balancer.BuildOptions{})
   201  	defer pb.Close()
   202  
   203  	t.Log("Two localities, with priorities [0, 1], each with one backend.")
   204  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   205  		ResolverState: resolver.State{
   206  			Addresses: []resolver.Address{
   207  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
   208  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
   209  			},
   210  		},
   211  		BalancerConfig: &LBConfig{
   212  			Children: map[string]*Child{
   213  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   214  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   215  			},
   216  			Priorities: []string{"child-0", "child-1"},
   217  		},
   218  	}); err != nil {
   219  		t.Fatalf("failed to update ClientConn state: %v", err)
   220  	}
   221  
   222  	addrs0 := <-cc.NewSubConnAddrsCh
   223  	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
   224  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   225  	}
   226  	sc0 := <-cc.NewSubConnCh
   227  
   228  	t.Log("Make p0 ready.")
   229  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   230  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   231  
   232  	// Test roundrobin with only p0 subconns.
   233  	if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil {
   234  		t.Fatal(err.Error())
   235  	}
   236  
   237  	t.Log("Turn down 0, will start and use 1.")
   238  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
   239  	// Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs
   240  	// will retry.
   241  	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
   242  		t.Fatal(err.Error())
   243  	}
   244  
   245  	t.Log("Handle SubConn creation from 1.")
   246  	addrs1 := <-cc.NewSubConnAddrsCh
   247  	if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want {
   248  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   249  	}
   250  	sc1 := <-cc.NewSubConnCh
   251  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   252  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   253  
   254  	// Test pick with 1.
   255  	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
   256  		t.Fatal(err.Error())
   257  	}
   258  
   259  	t.Log("Add p2, it shouldn't cause any updates.")
   260  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   261  		ResolverState: resolver.State{
   262  			Addresses: []resolver.Address{
   263  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
   264  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
   265  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-2"}),
   266  			},
   267  		},
   268  		BalancerConfig: &LBConfig{
   269  			Children: map[string]*Child{
   270  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   271  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   272  				"child-2": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   273  			},
   274  			Priorities: []string{"child-0", "child-1", "child-2"},
   275  		},
   276  	}); err != nil {
   277  		t.Fatalf("failed to update ClientConn state: %v", err)
   278  	}
   279  
   280  	select {
   281  	case sc := <-cc.NewSubConnCh:
   282  		t.Fatalf("got unexpected new SubConn, %s", sc)
   283  	case <-cc.ShutdownSubConnCh:
   284  		t.Fatalf("got unexpected shutdown SubConn")
   285  	case <-time.After(time.Millisecond * 100):
   286  	}
   287  
   288  	t.Log("Turn down 1, use 2.")
   289  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
   290  
   291  	// Before 2 gets READY, picker should return NoSubConnAvailable, so RPCs
   292  	// will retry.
   293  	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
   294  		t.Fatal(err.Error())
   295  	}
   296  
   297  	addrs2 := <-cc.NewSubConnAddrsCh
   298  	if got, want := addrs2[0].Addr, testBackendAddrStrs[2]; got != want {
   299  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   300  	}
   301  	sc2 := <-cc.NewSubConnCh
   302  	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   303  	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   304  
   305  	// Test pick with 2.
   306  	if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil {
   307  		t.Fatal(err.Error())
   308  	}
   309  
   310  	t.Log("Remove 2, use 1.")
   311  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   312  		ResolverState: resolver.State{
   313  			Addresses: []resolver.Address{
   314  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
   315  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
   316  			},
   317  		},
   318  		BalancerConfig: &LBConfig{
   319  			Children: map[string]*Child{
   320  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   321  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   322  			},
   323  			Priorities: []string{"child-0", "child-1"},
   324  		},
   325  	}); err != nil {
   326  		t.Fatalf("failed to update ClientConn state: %v", err)
   327  	}
   328  
   329  	// p2 SubConns are shut down.
   330  	scToShutdown := <-cc.ShutdownSubConnCh
   331  	if scToShutdown != sc2 {
   332  		t.Fatalf("ShutdownSubConn, want %v, got %v", sc2, scToShutdown)
   333  	}
   334  
   335  	// Should get an update with 1's old transient failure picker, to override
   336  	// 2's old picker.
   337  	if err := cc.WaitForErrPicker(ctx); err != nil {
   338  		t.Fatal(err.Error())
   339  	}
   340  	<-cc.NewStateCh // Drain to match picker
   341  
   342  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   343  	// Does not change the aggregate state, because round robin does not leave
   344  	// TRANSIENT_FAILURE if a subconn goes CONNECTING.
   345  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   346  
   347  	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
   348  		t.Fatal(err.Error())
   349  	}
   350  }
   351  
   352  // Lower priority is used when higher priority turns Connecting from Ready.
   353  // Because changing from Ready to Connecting is a failure.
   354  //
   355  // Init 0 and 1; 0 is up, use 0; 0 is connecting, 1 is up, use 1; 0 is ready,
   356  // use 0.
   357  func (s) TestPriority_HighPriorityToConnectingFromReady(t *testing.T) {
   358  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
   359  	defer cancel()
   360  
   361  	cc := testutils.NewBalancerClientConn(t)
   362  	bb := balancer.Get(Name)
   363  	pb := bb.Build(cc, balancer.BuildOptions{})
   364  	defer pb.Close()
   365  
   366  	// Two localities, with priorities [0, 1], each with one backend.
   367  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   368  		ResolverState: resolver.State{
   369  			Addresses: []resolver.Address{
   370  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
   371  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
   372  			},
   373  		},
   374  		BalancerConfig: &LBConfig{
   375  			Children: map[string]*Child{
   376  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   377  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   378  			},
   379  			Priorities: []string{"child-0", "child-1"},
   380  		},
   381  	}); err != nil {
   382  		t.Fatalf("failed to update ClientConn state: %v", err)
   383  	}
   384  
   385  	addrs0 := <-cc.NewSubConnAddrsCh
   386  	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
   387  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   388  	}
   389  	sc0 := <-cc.NewSubConnCh
   390  
   391  	// p0 is ready.
   392  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   393  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   394  
   395  	// Test roundrobin with only p0 subconns.
   396  	if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil {
   397  		t.Fatal(err.Error())
   398  	}
   399  
   400  	// Turn 0 to TransientFailure, will start and use 1.
   401  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
   402  
   403  	// Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs
   404  	// will retry.
   405  	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
   406  		t.Fatal(err.Error())
   407  	}
   408  
   409  	// Handle SubConn creation from 1.
   410  	addrs1 := <-cc.NewSubConnAddrsCh
   411  	if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want {
   412  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   413  	}
   414  	sc1 := <-cc.NewSubConnCh
   415  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   416  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   417  
   418  	// Test pick with 1.
   419  	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
   420  		t.Fatal(err.Error())
   421  	}
   422  
   423  	// Turn 0 back to Ready.
   424  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   425  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   426  
   427  	// p1 subconn should be shut down.
   428  	scToShutdown := <-cc.ShutdownSubConnCh
   429  	if scToShutdown != sc1 {
   430  		t.Fatalf("ShutdownSubConn, want %v, got %v", sc0, scToShutdown)
   431  	}
   432  
   433  	if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil {
   434  		t.Fatal(err.Error())
   435  	}
   436  }
   437  
   438  // Add a lower priority while the higher priority is down.
   439  //
   440  // Init 0 and 1; 0 and 1 both down; add 2, use 2.
   441  func (s) TestPriority_HigherDownWhileAddingLower(t *testing.T) {
   442  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
   443  	defer cancel()
   444  
   445  	cc := testutils.NewBalancerClientConn(t)
   446  	bb := balancer.Get(Name)
   447  	pb := bb.Build(cc, balancer.BuildOptions{})
   448  	defer pb.Close()
   449  
   450  	// Two localities, with different priorities, each with one backend.
   451  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   452  		ResolverState: resolver.State{
   453  			Addresses: []resolver.Address{
   454  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
   455  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
   456  			},
   457  		},
   458  		BalancerConfig: &LBConfig{
   459  			Children: map[string]*Child{
   460  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   461  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   462  			},
   463  			Priorities: []string{"child-0", "child-1"},
   464  		},
   465  	}); err != nil {
   466  		t.Fatalf("failed to update ClientConn state: %v", err)
   467  	}
   468  
   469  	addrs0 := <-cc.NewSubConnAddrsCh
   470  	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
   471  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   472  	}
   473  	sc0 := <-cc.NewSubConnCh
   474  
   475  	t.Log("Turn down 0, 1 is used.")
   476  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
   477  
   478  	// Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs
   479  	// will retry.
   480  	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
   481  		t.Fatal(err.Error())
   482  	}
   483  
   484  	addrs1 := <-cc.NewSubConnAddrsCh
   485  	if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want {
   486  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   487  	}
   488  	sc1 := <-cc.NewSubConnCh
   489  
   490  	t.Log("Turn down 1, pick should error.")
   491  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
   492  
   493  	// Test pick failure.
   494  	if err := cc.WaitForErrPicker(ctx); err != nil {
   495  		t.Fatal(err.Error())
   496  	}
   497  	<-cc.NewStateCh // Drain to match picker
   498  
   499  	t.Log("Add p2, it should create a new SubConn.")
   500  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   501  		ResolverState: resolver.State{
   502  			Addresses: []resolver.Address{
   503  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
   504  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
   505  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-2"}),
   506  			},
   507  		},
   508  		BalancerConfig: &LBConfig{
   509  			Children: map[string]*Child{
   510  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   511  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   512  				"child-2": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   513  			},
   514  			Priorities: []string{"child-0", "child-1", "child-2"},
   515  		},
   516  	}); err != nil {
   517  		t.Fatalf("failed to update ClientConn state: %v", err)
   518  	}
   519  
   520  	// A new connecting picker should be updated for the new priority.
   521  	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
   522  		t.Fatal(err.Error())
   523  	}
   524  
   525  	addrs2 := <-cc.NewSubConnAddrsCh
   526  	if got, want := addrs2[0].Addr, testBackendAddrStrs[2]; got != want {
   527  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   528  	}
   529  	sc2 := <-cc.NewSubConnCh
   530  	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   531  	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   532  
   533  	// Test pick with 2.
   534  	if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil {
   535  		t.Fatal(err.Error())
   536  	}
   537  }
   538  
   539  // When a higher priority becomes available, all lower priorities are closed.
   540  //
   541  // Init 0,1,2; 0 and 1 down, use 2; 0 up, close 1 and 2.
   542  func (s) TestPriority_HigherReadyCloseAllLower(t *testing.T) {
   543  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
   544  	defer cancel()
   545  
   546  	cc := testutils.NewBalancerClientConn(t)
   547  	bb := balancer.Get(Name)
   548  	pb := bb.Build(cc, balancer.BuildOptions{})
   549  	defer pb.Close()
   550  
   551  	// Three localities, with priorities [0,1,2], each with one backend.
   552  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   553  		ResolverState: resolver.State{
   554  			Addresses: []resolver.Address{
   555  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
   556  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
   557  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-2"}),
   558  			},
   559  		},
   560  		BalancerConfig: &LBConfig{
   561  			Children: map[string]*Child{
   562  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   563  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   564  				"child-2": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   565  			},
   566  			Priorities: []string{"child-0", "child-1", "child-2"},
   567  		},
   568  	}); err != nil {
   569  		t.Fatalf("failed to update ClientConn state: %v", err)
   570  	}
   571  
   572  	addrs0 := <-cc.NewSubConnAddrsCh
   573  	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
   574  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   575  	}
   576  	sc0 := <-cc.NewSubConnCh
   577  
   578  	// Turn down 0, 1 is used.
   579  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
   580  	// Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs
   581  	// will retry.
   582  	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
   583  		t.Fatal(err.Error())
   584  	}
   585  
   586  	addrs1 := <-cc.NewSubConnAddrsCh
   587  	if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want {
   588  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   589  	}
   590  	sc1 := <-cc.NewSubConnCh
   591  
   592  	// Turn down 1, 2 is used.
   593  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
   594  	// Before 2 gets READY, picker should return NoSubConnAvailable, so RPCs
   595  	// will retry.
   596  	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
   597  		t.Fatal(err.Error())
   598  	}
   599  
   600  	addrs2 := <-cc.NewSubConnAddrsCh
   601  	if got, want := addrs2[0].Addr, testBackendAddrStrs[2]; got != want {
   602  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   603  	}
   604  	sc2 := <-cc.NewSubConnCh
   605  	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   606  	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   607  
   608  	// Test pick with 2.
   609  	if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil {
   610  		t.Fatal(err.Error())
   611  	}
   612  
   613  	// When 0 becomes ready, 0 should be used, 1 and 2 should all be closed.
   614  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   615  
   616  	// sc1 and sc2 should be shut down.
   617  	//
   618  	// With localities caching, the lower priorities are closed after a timeout,
   619  	// in goroutines. The order is no longer guaranteed.
   620  	scToShutdown := []balancer.SubConn{<-cc.ShutdownSubConnCh, <-cc.ShutdownSubConnCh}
   621  	if !(scToShutdown[0] == sc1 && scToShutdown[1] == sc2) && !(scToShutdown[0] == sc2 && scToShutdown[1] == sc1) {
   622  		t.Errorf("ShutdownSubConn, want [%v, %v], got %v", sc1, sc2, scToShutdown)
   623  	}
   624  
   625  	// Test pick with 0.
   626  	if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil {
   627  		t.Fatal(err.Error())
   628  	}
   629  }
   630  
   631  // At init, start the next lower priority after timeout if the higher priority
   632  // doesn't get ready.
   633  //
   634  // Init 0,1; 0 is not ready (in connecting), after timeout, use 1.
   635  func (s) TestPriority_InitTimeout(t *testing.T) {
   636  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
   637  	defer cancel()
   638  
   639  	const testPriorityInitTimeout = 200 * time.Millisecond
   640  	defer func() func() {
   641  		old := DefaultPriorityInitTimeout
   642  		DefaultPriorityInitTimeout = testPriorityInitTimeout
   643  		return func() {
   644  			DefaultPriorityInitTimeout = old
   645  		}
   646  	}()()
   647  
   648  	cc := testutils.NewBalancerClientConn(t)
   649  	bb := balancer.Get(Name)
   650  	pb := bb.Build(cc, balancer.BuildOptions{})
   651  	defer pb.Close()
   652  
   653  	// Two localities, with different priorities, each with one backend.
   654  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   655  		ResolverState: resolver.State{
   656  			Addresses: []resolver.Address{
   657  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
   658  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
   659  			},
   660  		},
   661  		BalancerConfig: &LBConfig{
   662  			Children: map[string]*Child{
   663  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   664  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   665  			},
   666  			Priorities: []string{"child-0", "child-1"},
   667  		},
   668  	}); err != nil {
   669  		t.Fatalf("failed to update ClientConn state: %v", err)
   670  	}
   671  
   672  	addrs0 := <-cc.NewSubConnAddrsCh
   673  	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
   674  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   675  	}
   676  	sc0 := <-cc.NewSubConnCh
   677  
   678  	// Keep 0 in connecting, 1 will be used after init timeout.
   679  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   680  
   681  	// Make sure new SubConn is created before timeout.
   682  	select {
   683  	case <-time.After(testPriorityInitTimeout * 3 / 4):
   684  	case <-cc.NewSubConnAddrsCh:
   685  		t.Fatalf("Got a new SubConn too early (Within timeout). Expect a new SubConn only after timeout")
   686  	}
   687  
   688  	addrs1 := <-cc.NewSubConnAddrsCh
   689  	if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want {
   690  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   691  	}
   692  	sc1 := <-cc.NewSubConnCh
   693  
   694  	// After the init timer of p0, when switching to p1, a connecting picker
   695  	// will be sent to the parent. Clear it here.
   696  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   697  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   698  
   699  	// Test pick with 1.
   700  	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
   701  		t.Fatal(err.Error())
   702  	}
   703  }
   704  
   705  // EDS removes all priorities, and re-adds them.
   706  func (s) TestPriority_RemovesAllPriorities(t *testing.T) {
   707  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
   708  	defer cancel()
   709  
   710  	const testPriorityInitTimeout = 200 * time.Millisecond
   711  	defer func() func() {
   712  		old := DefaultPriorityInitTimeout
   713  		DefaultPriorityInitTimeout = testPriorityInitTimeout
   714  		return func() {
   715  			DefaultPriorityInitTimeout = old
   716  		}
   717  	}()()
   718  
   719  	cc := testutils.NewBalancerClientConn(t)
   720  	bb := balancer.Get(Name)
   721  	pb := bb.Build(cc, balancer.BuildOptions{})
   722  	defer pb.Close()
   723  
   724  	// Two localities, with different priorities, each with one backend.
   725  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   726  		ResolverState: resolver.State{
   727  			Addresses: []resolver.Address{
   728  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
   729  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
   730  			},
   731  		},
   732  		BalancerConfig: &LBConfig{
   733  			Children: map[string]*Child{
   734  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   735  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   736  			},
   737  			Priorities: []string{"child-0", "child-1"},
   738  		},
   739  	}); err != nil {
   740  		t.Fatalf("failed to update ClientConn state: %v", err)
   741  	}
   742  
   743  	addrs0 := <-cc.NewSubConnAddrsCh
   744  	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
   745  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   746  	}
   747  	sc0 := <-cc.NewSubConnCh
   748  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   749  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   750  
   751  	// Test roundrobin with only p0 subconns.
   752  	if err := cc.WaitForRoundRobinPicker(ctx, sc0); err != nil {
   753  		t.Fatal(err.Error())
   754  	}
   755  
   756  	// Remove all priorities.
   757  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   758  		ResolverState: resolver.State{
   759  			Addresses: nil,
   760  		},
   761  		BalancerConfig: &LBConfig{
   762  			Children:   nil,
   763  			Priorities: nil,
   764  		},
   765  	}); err != nil {
   766  		t.Fatalf("failed to update ClientConn state: %v", err)
   767  	}
   768  
   769  	// p0 subconn should be shut down.
   770  	scToShutdown := <-cc.ShutdownSubConnCh
   771  	if scToShutdown != sc0 {
   772  		t.Fatalf("ShutdownSubConn, want %v, got %v", sc0, scToShutdown)
   773  	}
   774  
   775  	// Test pick return TransientFailure.
   776  	if err := cc.WaitForPickerWithErr(ctx, ErrAllPrioritiesRemoved); err != nil {
   777  		t.Fatal(err.Error())
   778  	}
   779  
   780  	// Re-add two localities, with previous priorities, but different backends.
   781  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   782  		ResolverState: resolver.State{
   783  			Addresses: []resolver.Address{
   784  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-0"}),
   785  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[3]}, []string{"child-1"}),
   786  			},
   787  		},
   788  		BalancerConfig: &LBConfig{
   789  			Children: map[string]*Child{
   790  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   791  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   792  			},
   793  			Priorities: []string{"child-0", "child-1"},
   794  		},
   795  	}); err != nil {
   796  		t.Fatalf("failed to update ClientConn state: %v", err)
   797  	}
   798  
   799  	addrs01 := <-cc.NewSubConnAddrsCh
   800  	if got, want := addrs01[0].Addr, testBackendAddrStrs[2]; got != want {
   801  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   802  	}
   803  	sc01 := <-cc.NewSubConnCh
   804  
   805  	// Don't send any update to p0, so to not override the old state of p0.
   806  	// Later, connect to p1 and then remove p1. This will fallback to p0, and
   807  	// will send p0's old picker if they are not correctly removed.
   808  
   809  	// p1 will be used after priority init timeout.
   810  	addrs11 := <-cc.NewSubConnAddrsCh
   811  	if got, want := addrs11[0].Addr, testBackendAddrStrs[3]; got != want {
   812  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   813  	}
   814  	sc11 := <-cc.NewSubConnCh
   815  	sc11.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   816  	sc11.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   817  
   818  	// Test roundrobin with only p1 subconns.
   819  	if err := cc.WaitForRoundRobinPicker(ctx, sc11); err != nil {
   820  		t.Fatal(err.Error())
   821  	}
   822  
   823  	// Remove p1, to fallback to p0.
   824  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   825  		ResolverState: resolver.State{
   826  			Addresses: []resolver.Address{
   827  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-0"}),
   828  			},
   829  		},
   830  		BalancerConfig: &LBConfig{
   831  			Children: map[string]*Child{
   832  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   833  			},
   834  			Priorities: []string{"child-0"},
   835  		},
   836  	}); err != nil {
   837  		t.Fatalf("failed to update ClientConn state: %v", err)
   838  	}
   839  
   840  	// p1 subconn should be shut down.
   841  	scToShutdown1 := <-cc.ShutdownSubConnCh
   842  	if scToShutdown1 != sc11 {
   843  		t.Fatalf("ShutdownSubConn, want %v, got %v", sc11, scToShutdown1)
   844  	}
   845  
   846  	// Test pick return NoSubConn.
   847  	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
   848  		t.Fatal(err.Error())
   849  	}
   850  
   851  	// Send an ready update for the p0 sc that was received when re-adding
   852  	// priorities.
   853  	sc01.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   854  	sc01.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   855  
   856  	// Test roundrobin with only p0 subconns.
   857  	if err := cc.WaitForRoundRobinPicker(ctx, sc01); err != nil {
   858  		t.Fatal(err.Error())
   859  	}
   860  
   861  	select {
   862  	case <-cc.NewPickerCh:
   863  		t.Fatalf("got unexpected new picker")
   864  	case <-cc.NewSubConnCh:
   865  		t.Fatalf("got unexpected new SubConn")
   866  	case <-cc.ShutdownSubConnCh:
   867  		t.Fatalf("got unexpected shutdown SubConn")
   868  	case <-time.After(time.Millisecond * 100):
   869  	}
   870  }
   871  
   872  // Test the case where the high priority contains no backends. The low priority
   873  // will be used.
   874  func (s) TestPriority_HighPriorityNoEndpoints(t *testing.T) {
   875  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
   876  	defer cancel()
   877  
   878  	cc := testutils.NewBalancerClientConn(t)
   879  	bb := balancer.Get(Name)
   880  	pb := bb.Build(cc, balancer.BuildOptions{})
   881  	defer pb.Close()
   882  
   883  	// Two localities, with priorities [0, 1], each with one backend.
   884  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   885  		ResolverState: resolver.State{
   886  			Addresses: []resolver.Address{
   887  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
   888  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
   889  			},
   890  		},
   891  		BalancerConfig: &LBConfig{
   892  			Children: map[string]*Child{
   893  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   894  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   895  			},
   896  			Priorities: []string{"child-0", "child-1"},
   897  		},
   898  	}); err != nil {
   899  		t.Fatalf("failed to update ClientConn state: %v", err)
   900  	}
   901  
   902  	addrs1 := <-cc.NewSubConnAddrsCh
   903  	if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want {
   904  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   905  	}
   906  	sc1 := <-cc.NewSubConnCh
   907  
   908  	// p0 is ready.
   909  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   910  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   911  
   912  	// Test roundrobin with only p0 subconns.
   913  	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
   914  		t.Fatal(err.Error())
   915  	}
   916  
   917  	// Remove addresses from priority 0, should use p1.
   918  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   919  		ResolverState: resolver.State{
   920  			Addresses: []resolver.Address{
   921  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
   922  			},
   923  		},
   924  		BalancerConfig: &LBConfig{
   925  			Children: map[string]*Child{
   926  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   927  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   928  			},
   929  			Priorities: []string{"child-0", "child-1"},
   930  		},
   931  	}); err != nil {
   932  		t.Fatalf("failed to update ClientConn state: %v", err)
   933  	}
   934  
   935  	// p0 will shutdown the subconn, and ClientConn will send a sc update to
   936  	// shutdown.
   937  	scToShutdown := <-cc.ShutdownSubConnCh
   938  	scToShutdown.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Shutdown})
   939  
   940  	addrs2 := <-cc.NewSubConnAddrsCh
   941  	if got, want := addrs2[0].Addr, testBackendAddrStrs[1]; got != want {
   942  		t.Fatalf("sc is created with addr %v, want %v", got, want)
   943  	}
   944  	sc2 := <-cc.NewSubConnCh
   945  
   946  	// Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs
   947  	// will retry.
   948  	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
   949  		t.Fatal(err.Error())
   950  	}
   951  
   952  	// p1 is ready.
   953  	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
   954  	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
   955  
   956  	// Test roundrobin with only p1 subconns.
   957  	if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil {
   958  		t.Fatal(err.Error())
   959  	}
   960  }
   961  
   962  // Test the case where the first and only priority is removed.
   963  func (s) TestPriority_FirstPriorityUnavailable(t *testing.T) {
   964  	const testPriorityInitTimeout = 200 * time.Millisecond
   965  	defer func(t time.Duration) {
   966  		DefaultPriorityInitTimeout = t
   967  	}(DefaultPriorityInitTimeout)
   968  	DefaultPriorityInitTimeout = testPriorityInitTimeout
   969  
   970  	cc := testutils.NewBalancerClientConn(t)
   971  	bb := balancer.Get(Name)
   972  	pb := bb.Build(cc, balancer.BuildOptions{})
   973  	defer pb.Close()
   974  
   975  	// One localities, with priorities [0], each with one backend.
   976  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   977  		ResolverState: resolver.State{
   978  			Addresses: []resolver.Address{
   979  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
   980  			},
   981  		},
   982  		BalancerConfig: &LBConfig{
   983  			Children: map[string]*Child{
   984  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
   985  			},
   986  			Priorities: []string{"child-0"},
   987  		},
   988  	}); err != nil {
   989  		t.Fatalf("failed to update ClientConn state: %v", err)
   990  	}
   991  
   992  	// Remove the only localities.
   993  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
   994  		ResolverState: resolver.State{
   995  			Addresses: nil,
   996  		},
   997  		BalancerConfig: &LBConfig{
   998  			Children:   nil,
   999  			Priorities: nil,
  1000  		},
  1001  	}); err != nil {
  1002  		t.Fatalf("failed to update ClientConn state: %v", err)
  1003  	}
  1004  
  1005  	// Wait after double the init timer timeout, to ensure it doesn't panic.
  1006  	time.Sleep(testPriorityInitTimeout * 2)
  1007  }
  1008  
  1009  // When a child is moved from low priority to high.
  1010  //
  1011  // Init a(p0) and b(p1); a(p0) is up, use a; move b to p0, a to p1, use b.
  1012  func (s) TestPriority_MoveChildToHigherPriority(t *testing.T) {
  1013  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
  1014  	defer cancel()
  1015  
  1016  	cc := testutils.NewBalancerClientConn(t)
  1017  	bb := balancer.Get(Name)
  1018  	pb := bb.Build(cc, balancer.BuildOptions{})
  1019  	defer pb.Close()
  1020  
  1021  	// Two children, with priorities [0, 1], each with one backend.
  1022  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1023  		ResolverState: resolver.State{
  1024  			Addresses: []resolver.Address{
  1025  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1026  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
  1027  			},
  1028  		},
  1029  		BalancerConfig: &LBConfig{
  1030  			Children: map[string]*Child{
  1031  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1032  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1033  			},
  1034  			Priorities: []string{"child-0", "child-1"},
  1035  		},
  1036  	}); err != nil {
  1037  		t.Fatalf("failed to update ClientConn state: %v", err)
  1038  	}
  1039  
  1040  	addrs1 := <-cc.NewSubConnAddrsCh
  1041  	if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want {
  1042  		t.Fatalf("sc is created with addr %v, want %v", got, want)
  1043  	}
  1044  	sc1 := <-cc.NewSubConnCh
  1045  
  1046  	// p0 is ready.
  1047  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
  1048  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
  1049  
  1050  	// Test roundrobin with only p0 subconns.
  1051  	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
  1052  		t.Fatal(err.Error())
  1053  	}
  1054  
  1055  	// Swap child with p0 and p1, the child at lower priority should now be the
  1056  	// higher priority, and be used. The old SubConn should be closed.
  1057  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1058  		ResolverState: resolver.State{
  1059  			Addresses: []resolver.Address{
  1060  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1061  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
  1062  			},
  1063  		},
  1064  		BalancerConfig: &LBConfig{
  1065  			Children: map[string]*Child{
  1066  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1067  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1068  			},
  1069  			Priorities: []string{"child-1", "child-0"},
  1070  		},
  1071  	}); err != nil {
  1072  		t.Fatalf("failed to update ClientConn state: %v", err)
  1073  	}
  1074  
  1075  	// When the new child for p0 is changed from the previous child, the
  1076  	// balancer should immediately update the picker so the picker from old
  1077  	// child is not used. In this case, the picker becomes a
  1078  	// no-subconn-available picker because this child is just started.
  1079  	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
  1080  		t.Fatal(err.Error())
  1081  	}
  1082  
  1083  	// Old subconn should be shut down.
  1084  	scToShutdown := <-cc.ShutdownSubConnCh
  1085  	if scToShutdown != sc1 {
  1086  		t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, scToShutdown)
  1087  	}
  1088  
  1089  	addrs2 := <-cc.NewSubConnAddrsCh
  1090  	if got, want := addrs2[0].Addr, testBackendAddrStrs[1]; got != want {
  1091  		t.Fatalf("sc is created with addr %v, want %v", got, want)
  1092  	}
  1093  	sc2 := <-cc.NewSubConnCh
  1094  
  1095  	// New p0 child is ready.
  1096  	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
  1097  	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
  1098  
  1099  	// Test roundrobin with only new subconns.
  1100  	if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil {
  1101  		t.Fatal(err.Error())
  1102  	}
  1103  }
  1104  
  1105  // When a child is in lower priority, and in use (because higher is down),
  1106  // move it from low priority to high.
  1107  //
  1108  // Init a(p0) and b(p1); a(p0) is down, use b; move b to p0, a to p1, use b.
  1109  func (s) TestPriority_MoveReadyChildToHigherPriority(t *testing.T) {
  1110  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
  1111  	defer cancel()
  1112  
  1113  	cc := testutils.NewBalancerClientConn(t)
  1114  	bb := balancer.Get(Name)
  1115  	pb := bb.Build(cc, balancer.BuildOptions{})
  1116  	defer pb.Close()
  1117  
  1118  	// Two children, with priorities [0, 1], each with one backend.
  1119  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1120  		ResolverState: resolver.State{
  1121  			Addresses: []resolver.Address{
  1122  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1123  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
  1124  			},
  1125  		},
  1126  		BalancerConfig: &LBConfig{
  1127  			Children: map[string]*Child{
  1128  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1129  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1130  			},
  1131  			Priorities: []string{"child-0", "child-1"},
  1132  		},
  1133  	}); err != nil {
  1134  		t.Fatalf("failed to update ClientConn state: %v", err)
  1135  	}
  1136  
  1137  	addrs0 := <-cc.NewSubConnAddrsCh
  1138  	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
  1139  		t.Fatalf("sc is created with addr %v, want %v", got, want)
  1140  	}
  1141  	sc0 := <-cc.NewSubConnCh
  1142  
  1143  	// p0 is down.
  1144  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
  1145  	// Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs
  1146  	// will retry.
  1147  	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
  1148  		t.Fatal(err.Error())
  1149  	}
  1150  
  1151  	addrs1 := <-cc.NewSubConnAddrsCh
  1152  	if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want {
  1153  		t.Fatalf("sc is created with addr %v, want %v", got, want)
  1154  	}
  1155  	sc1 := <-cc.NewSubConnCh
  1156  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
  1157  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
  1158  
  1159  	// Test roundrobin with only p1 subconns.
  1160  	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
  1161  		t.Fatal(err.Error())
  1162  	}
  1163  
  1164  	// Swap child with p0 and p1, the child at lower priority should now be the
  1165  	// higher priority, and be used. The old SubConn should be closed.
  1166  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1167  		ResolverState: resolver.State{
  1168  			Addresses: []resolver.Address{
  1169  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1170  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
  1171  			},
  1172  		},
  1173  		BalancerConfig: &LBConfig{
  1174  			Children: map[string]*Child{
  1175  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1176  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1177  			},
  1178  			Priorities: []string{"child-1", "child-0"},
  1179  		},
  1180  	}); err != nil {
  1181  		t.Fatalf("failed to update ClientConn state: %v", err)
  1182  	}
  1183  
  1184  	// Old subconn from child-0 should be removed.
  1185  	scToShutdown := <-cc.ShutdownSubConnCh
  1186  	if scToShutdown != sc0 {
  1187  		t.Fatalf("ShutdownSubConn, want %v, got %v", sc0, scToShutdown)
  1188  	}
  1189  
  1190  	// Because this was a ready child moved to a higher priority, no new subconn
  1191  	// or picker should be updated.
  1192  	select {
  1193  	case <-cc.NewSubConnCh:
  1194  		t.Fatalf("got unexpected new SubConn")
  1195  	case <-cc.ShutdownSubConnCh:
  1196  		t.Fatalf("got unexpected shutdown SubConn")
  1197  	case <-time.After(time.Millisecond * 100):
  1198  	}
  1199  }
  1200  
  1201  // When the lowest child is in use, and is removed, should use the higher
  1202  // priority child even though it's not ready.
  1203  //
  1204  // Init a(p0) and b(p1); a(p0) is down, use b; move b to p0, a to p1, use b.
  1205  func (s) TestPriority_RemoveReadyLowestChild(t *testing.T) {
  1206  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
  1207  	defer cancel()
  1208  
  1209  	cc := testutils.NewBalancerClientConn(t)
  1210  	bb := balancer.Get(Name)
  1211  	pb := bb.Build(cc, balancer.BuildOptions{})
  1212  	defer pb.Close()
  1213  
  1214  	// Two children, with priorities [0, 1], each with one backend.
  1215  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1216  		ResolverState: resolver.State{
  1217  			Addresses: []resolver.Address{
  1218  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1219  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
  1220  			},
  1221  		},
  1222  		BalancerConfig: &LBConfig{
  1223  			Children: map[string]*Child{
  1224  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1225  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1226  			},
  1227  			Priorities: []string{"child-0", "child-1"},
  1228  		},
  1229  	}); err != nil {
  1230  		t.Fatalf("failed to update ClientConn state: %v", err)
  1231  	}
  1232  
  1233  	addrs0 := <-cc.NewSubConnAddrsCh
  1234  	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
  1235  		t.Fatalf("sc is created with addr %v, want %v", got, want)
  1236  	}
  1237  	sc0 := <-cc.NewSubConnCh
  1238  
  1239  	// p0 is down.
  1240  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
  1241  	// Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs
  1242  	// will retry.
  1243  	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
  1244  		t.Fatal(err.Error())
  1245  	}
  1246  
  1247  	addrs1 := <-cc.NewSubConnAddrsCh
  1248  	if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want {
  1249  		t.Fatalf("sc is created with addr %v, want %v", got, want)
  1250  	}
  1251  	sc1 := <-cc.NewSubConnCh
  1252  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
  1253  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
  1254  
  1255  	// Test roundrobin with only p1 subconns.
  1256  	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
  1257  		t.Fatal(err.Error())
  1258  	}
  1259  
  1260  	// Remove child with p1, the child at higher priority should now be used.
  1261  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1262  		ResolverState: resolver.State{
  1263  			Addresses: []resolver.Address{
  1264  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1265  			},
  1266  		},
  1267  		BalancerConfig: &LBConfig{
  1268  			Children: map[string]*Child{
  1269  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1270  			},
  1271  			Priorities: []string{"child-0"},
  1272  		},
  1273  	}); err != nil {
  1274  		t.Fatalf("failed to update ClientConn state: %v", err)
  1275  	}
  1276  
  1277  	// Old subconn from child-1 should be shut down.
  1278  	scToShutdown := <-cc.ShutdownSubConnCh
  1279  	if scToShutdown != sc1 {
  1280  		t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, scToShutdown)
  1281  	}
  1282  
  1283  	if err := cc.WaitForErrPicker(ctx); err != nil {
  1284  		t.Fatal(err.Error())
  1285  	}
  1286  	<-cc.NewStateCh // Drain to match picker
  1287  
  1288  	// Because there was no new child, no new subconn should be created.
  1289  	select {
  1290  	case <-cc.NewSubConnCh:
  1291  		t.Fatalf("got unexpected new SubConn")
  1292  	case <-time.After(time.Millisecond * 100):
  1293  	}
  1294  }
  1295  
  1296  // When a ready child is removed, it's kept in cache. Re-adding doesn't create subconns.
  1297  //
  1298  // Init 0; 0 is up, use 0; remove 0, only picker is updated, no subconn is
  1299  // removed; re-add 0, picker is updated.
  1300  func (s) TestPriority_ReadyChildRemovedButInCache(t *testing.T) {
  1301  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
  1302  	defer cancel()
  1303  
  1304  	const testChildCacheTimeout = time.Second
  1305  	defer func() func() {
  1306  		old := DefaultSubBalancerCloseTimeout
  1307  		DefaultSubBalancerCloseTimeout = testChildCacheTimeout
  1308  		return func() {
  1309  			DefaultSubBalancerCloseTimeout = old
  1310  		}
  1311  	}()()
  1312  
  1313  	cc := testutils.NewBalancerClientConn(t)
  1314  	bb := balancer.Get(Name)
  1315  	pb := bb.Build(cc, balancer.BuildOptions{})
  1316  	defer pb.Close()
  1317  
  1318  	// One children, with priorities [0], with one backend.
  1319  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1320  		ResolverState: resolver.State{
  1321  			Addresses: []resolver.Address{
  1322  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1323  			},
  1324  		},
  1325  		BalancerConfig: &LBConfig{
  1326  			Children: map[string]*Child{
  1327  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1328  			},
  1329  			Priorities: []string{"child-0"},
  1330  		},
  1331  	}); err != nil {
  1332  		t.Fatalf("failed to update ClientConn state: %v", err)
  1333  	}
  1334  
  1335  	addrs1 := <-cc.NewSubConnAddrsCh
  1336  	if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want {
  1337  		t.Fatalf("sc is created with addr %v, want %v", got, want)
  1338  	}
  1339  	sc1 := <-cc.NewSubConnCh
  1340  
  1341  	// p0 is ready.
  1342  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
  1343  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
  1344  
  1345  	// Test roundrobin with only p0 subconns.
  1346  	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
  1347  		t.Fatal(err.Error())
  1348  	}
  1349  
  1350  	// Remove the child, it shouldn't cause any conn changed, but picker should
  1351  	// be different.
  1352  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1353  		ResolverState:  resolver.State{},
  1354  		BalancerConfig: &LBConfig{},
  1355  	}); err != nil {
  1356  		t.Fatalf("failed to update ClientConn state: %v", err)
  1357  	}
  1358  
  1359  	if err := cc.WaitForPickerWithErr(ctx, ErrAllPrioritiesRemoved); err != nil {
  1360  		t.Fatal(err.Error())
  1361  	}
  1362  
  1363  	// But no conn changes should happen. Child balancer is in cache.
  1364  	select {
  1365  	case sc := <-cc.NewSubConnCh:
  1366  		t.Fatalf("got unexpected new SubConn: %s", sc)
  1367  	case sc := <-cc.ShutdownSubConnCh:
  1368  		t.Fatalf("got unexpected shutdown SubConn: %v", sc)
  1369  	case <-time.After(time.Millisecond * 100):
  1370  	}
  1371  
  1372  	// Re-add the child, shouldn't create new connections.
  1373  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1374  		ResolverState: resolver.State{
  1375  			Addresses: []resolver.Address{
  1376  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1377  			},
  1378  		},
  1379  		BalancerConfig: &LBConfig{
  1380  			Children: map[string]*Child{
  1381  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1382  			},
  1383  			Priorities: []string{"child-0"},
  1384  		},
  1385  	}); err != nil {
  1386  		t.Fatalf("failed to update ClientConn state: %v", err)
  1387  	}
  1388  
  1389  	// Test roundrobin with only p0 subconns.
  1390  	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
  1391  		t.Fatal(err.Error())
  1392  	}
  1393  
  1394  	// But no conn changes should happen. Child balancer is just taken out from
  1395  	// the cache.
  1396  	select {
  1397  	case sc := <-cc.NewSubConnCh:
  1398  		t.Fatalf("got unexpected new SubConn: %s", sc)
  1399  	case sc := <-cc.ShutdownSubConnCh:
  1400  		t.Fatalf("got unexpected shutdown SubConn: %v", sc)
  1401  	case <-time.After(time.Millisecond * 100):
  1402  	}
  1403  }
  1404  
  1405  // When the policy of a child is changed.
  1406  //
  1407  // Init 0; 0 is up, use 0; change 0's policy, 0 is used.
  1408  func (s) TestPriority_ChildPolicyChange(t *testing.T) {
  1409  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
  1410  	defer cancel()
  1411  
  1412  	cc := testutils.NewBalancerClientConn(t)
  1413  	bb := balancer.Get(Name)
  1414  	pb := bb.Build(cc, balancer.BuildOptions{})
  1415  	defer pb.Close()
  1416  
  1417  	// One children, with priorities [0], with one backend.
  1418  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1419  		ResolverState: resolver.State{
  1420  			Addresses: []resolver.Address{
  1421  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1422  			},
  1423  		},
  1424  		BalancerConfig: &LBConfig{
  1425  			Children: map[string]*Child{
  1426  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1427  			},
  1428  			Priorities: []string{"child-0"},
  1429  		},
  1430  	}); err != nil {
  1431  		t.Fatalf("failed to update ClientConn state: %v", err)
  1432  	}
  1433  
  1434  	addrs1 := <-cc.NewSubConnAddrsCh
  1435  	if got, want := addrs1[0].Addr, testBackendAddrStrs[0]; got != want {
  1436  		t.Fatalf("sc is created with addr %v, want %v", got, want)
  1437  	}
  1438  	sc1 := <-cc.NewSubConnCh
  1439  
  1440  	// p0 is ready.
  1441  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
  1442  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
  1443  
  1444  	// Test roundrobin with only p0 subconns.
  1445  	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
  1446  		t.Fatal(err.Error())
  1447  	}
  1448  
  1449  	// Change the policy for the child (still roundrobin, but with a different
  1450  	// name).
  1451  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1452  		ResolverState: resolver.State{
  1453  			Addresses: []resolver.Address{
  1454  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1455  			},
  1456  		},
  1457  		BalancerConfig: &LBConfig{
  1458  			Children: map[string]*Child{
  1459  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: testRRBalancerName}},
  1460  			},
  1461  			Priorities: []string{"child-0"},
  1462  		},
  1463  	}); err != nil {
  1464  		t.Fatalf("failed to update ClientConn state: %v", err)
  1465  	}
  1466  
  1467  	// Old subconn should be shut down.
  1468  	scToShutdown := <-cc.ShutdownSubConnCh
  1469  	if scToShutdown != sc1 {
  1470  		t.Fatalf("ShutdownSubConn, want %v, got %v", sc1, scToShutdown)
  1471  	}
  1472  
  1473  	// A new subconn should be created.
  1474  	addrs2 := <-cc.NewSubConnAddrsCh
  1475  	if got, want := addrs2[0].Addr, testBackendAddrStrs[0]; got != want {
  1476  		t.Fatalf("sc is created with addr %v, want %v", got, want)
  1477  	}
  1478  	sc2 := <-cc.NewSubConnCh
  1479  	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
  1480  	sc2.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
  1481  
  1482  	// Test pickfirst with the new subconns.
  1483  	if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil {
  1484  		t.Fatal(err.Error())
  1485  	}
  1486  }
  1487  
  1488  const inlineUpdateBalancerName = "test-inline-update-balancer"
  1489  
  1490  var errTestInlineStateUpdate = fmt.Errorf("don't like addresses, empty or not")
  1491  
  1492  func init() {
  1493  	stub.Register(inlineUpdateBalancerName, stub.BalancerFuncs{
  1494  		UpdateClientConnState: func(bd *stub.BalancerData, opts balancer.ClientConnState) error {
  1495  			bd.ClientConn.UpdateState(balancer.State{
  1496  				ConnectivityState: connectivity.Ready,
  1497  				Picker:            &testutils.TestConstPicker{Err: errTestInlineStateUpdate},
  1498  			})
  1499  			return nil
  1500  		},
  1501  	})
  1502  }
  1503  
  1504  // When the child policy update picker inline in a handleClientUpdate call
  1505  // (e.g., roundrobin handling empty addresses). There could be deadlock caused
  1506  // by acquiring a locked mutex.
  1507  func (s) TestPriority_ChildPolicyUpdatePickerInline(t *testing.T) {
  1508  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
  1509  	defer cancel()
  1510  
  1511  	cc := testutils.NewBalancerClientConn(t)
  1512  	bb := balancer.Get(Name)
  1513  	pb := bb.Build(cc, balancer.BuildOptions{})
  1514  	defer pb.Close()
  1515  
  1516  	// One children, with priorities [0], with one backend.
  1517  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1518  		ResolverState: resolver.State{
  1519  			Addresses: []resolver.Address{
  1520  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1521  			},
  1522  		},
  1523  		BalancerConfig: &LBConfig{
  1524  			Children: map[string]*Child{
  1525  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: inlineUpdateBalancerName}},
  1526  			},
  1527  			Priorities: []string{"child-0"},
  1528  		},
  1529  	}); err != nil {
  1530  		t.Fatalf("failed to update ClientConn state: %v", err)
  1531  	}
  1532  
  1533  	if err := cc.WaitForPickerWithErr(ctx, errTestInlineStateUpdate); err != nil {
  1534  		t.Fatal(err.Error())
  1535  	}
  1536  }
  1537  
  1538  // TestPriority_IgnoreReresolutionRequest tests the case where the priority
  1539  // policy has a single child policy. The test verifies that ResolveNow() calls
  1540  // from the child policy are ignored based on the value of the
  1541  // IgnoreReresolutionRequests field in the configuration.
  1542  func (s) TestPriority_IgnoreReresolutionRequest(t *testing.T) {
  1543  	// Register a stub balancer to act the child policy of the priority policy.
  1544  	// Provide an init function to the stub balancer to capture the ClientConn
  1545  	// passed to the child policy.
  1546  	ccCh := testutils.NewChannel()
  1547  	childPolicyName := t.Name()
  1548  	stub.Register(childPolicyName, stub.BalancerFuncs{
  1549  		Init: func(data *stub.BalancerData) {
  1550  			ccCh.Send(data.ClientConn)
  1551  		},
  1552  	})
  1553  
  1554  	cc := testutils.NewBalancerClientConn(t)
  1555  	bb := balancer.Get(Name)
  1556  	pb := bb.Build(cc, balancer.BuildOptions{})
  1557  	defer pb.Close()
  1558  
  1559  	// One children, with priorities [0], with one backend, reresolution is
  1560  	// ignored.
  1561  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1562  		ResolverState: resolver.State{
  1563  			Addresses: []resolver.Address{
  1564  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1565  			},
  1566  		},
  1567  		BalancerConfig: &LBConfig{
  1568  			Children: map[string]*Child{
  1569  				"child-0": {
  1570  					Config:                     &internalserviceconfig.BalancerConfig{Name: childPolicyName},
  1571  					IgnoreReresolutionRequests: true,
  1572  				},
  1573  			},
  1574  			Priorities: []string{"child-0"},
  1575  		},
  1576  	}); err != nil {
  1577  		t.Fatalf("failed to update ClientConn state: %v", err)
  1578  	}
  1579  
  1580  	// Retrieve the ClientConn passed to the child policy.
  1581  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
  1582  	defer cancel()
  1583  	val, err := ccCh.Receive(ctx)
  1584  	if err != nil {
  1585  		t.Fatalf("timeout waiting for ClientConn from the child policy")
  1586  	}
  1587  	balancerCC := val.(balancer.ClientConn)
  1588  
  1589  	// Since IgnoreReresolutionRequests was set to true, all ResolveNow() calls
  1590  	// should be ignored.
  1591  	for i := 0; i < 5; i++ {
  1592  		balancerCC.ResolveNow(resolver.ResolveNowOptions{})
  1593  	}
  1594  	select {
  1595  	case <-cc.ResolveNowCh:
  1596  		t.Fatalf("got unexpected ResolveNow() call")
  1597  	case <-time.After(defaultTestShortTimeout):
  1598  	}
  1599  
  1600  	// Send another update to set IgnoreReresolutionRequests to false.
  1601  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1602  		ResolverState: resolver.State{
  1603  			Addresses: []resolver.Address{
  1604  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1605  			},
  1606  		},
  1607  		BalancerConfig: &LBConfig{
  1608  			Children: map[string]*Child{
  1609  				"child-0": {
  1610  					Config:                     &internalserviceconfig.BalancerConfig{Name: childPolicyName},
  1611  					IgnoreReresolutionRequests: false,
  1612  				},
  1613  			},
  1614  			Priorities: []string{"child-0"},
  1615  		},
  1616  	}); err != nil {
  1617  		t.Fatalf("failed to update ClientConn state: %v", err)
  1618  	}
  1619  
  1620  	// Call ResolveNow() on the CC, it should be forwarded.
  1621  	balancerCC.ResolveNow(resolver.ResolveNowOptions{})
  1622  	select {
  1623  	case <-cc.ResolveNowCh:
  1624  	case <-time.After(time.Second):
  1625  		t.Fatalf("timeout waiting for ResolveNow()")
  1626  	}
  1627  
  1628  }
  1629  
  1630  // TestPriority_IgnoreReresolutionRequestTwoChildren tests the case where the
  1631  // priority policy has two child policies, one of them has the
  1632  // IgnoreReresolutionRequests field set to true while the other one has it set
  1633  // to false. The test verifies that ResolveNow() calls from the child which is
  1634  // set to ignore reresolution requests are ignored, while calls from the other
  1635  // child are processed.
  1636  func (s) TestPriority_IgnoreReresolutionRequestTwoChildren(t *testing.T) {
  1637  	// Register a stub balancer to act the child policy of the priority policy.
  1638  	// Provide an init function to the stub balancer to capture the ClientConn
  1639  	// passed to the child policy.
  1640  	ccCh := testutils.NewChannel()
  1641  	childPolicyName := t.Name()
  1642  	stub.Register(childPolicyName, stub.BalancerFuncs{
  1643  		Init: func(bd *stub.BalancerData) {
  1644  			ccCh.Send(bd.ClientConn)
  1645  			bd.Data = balancer.Get(roundrobin.Name).Build(bd.ClientConn, bd.BuildOptions)
  1646  		},
  1647  		UpdateClientConnState: func(bd *stub.BalancerData, ccs balancer.ClientConnState) error {
  1648  			bal := bd.Data.(balancer.Balancer)
  1649  			return bal.UpdateClientConnState(ccs)
  1650  		},
  1651  	})
  1652  
  1653  	cc := testutils.NewBalancerClientConn(t)
  1654  	bb := balancer.Get(Name)
  1655  	pb := bb.Build(cc, balancer.BuildOptions{})
  1656  	defer pb.Close()
  1657  
  1658  	// One children, with priorities [0, 1], each with one backend.
  1659  	// Reresolution is ignored for p0.
  1660  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1661  		ResolverState: resolver.State{
  1662  			Addresses: []resolver.Address{
  1663  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1664  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
  1665  			},
  1666  		},
  1667  		BalancerConfig: &LBConfig{
  1668  			Children: map[string]*Child{
  1669  				"child-0": {
  1670  					Config:                     &internalserviceconfig.BalancerConfig{Name: childPolicyName},
  1671  					IgnoreReresolutionRequests: true,
  1672  				},
  1673  				"child-1": {
  1674  					Config: &internalserviceconfig.BalancerConfig{Name: childPolicyName},
  1675  				},
  1676  			},
  1677  			Priorities: []string{"child-0", "child-1"},
  1678  		},
  1679  	}); err != nil {
  1680  		t.Fatalf("failed to update ClientConn state: %v", err)
  1681  	}
  1682  
  1683  	// Retrieve the ClientConn passed to the child policy from p0.
  1684  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
  1685  	defer cancel()
  1686  	val, err := ccCh.Receive(ctx)
  1687  	if err != nil {
  1688  		t.Fatalf("timeout waiting for ClientConn from the child policy")
  1689  	}
  1690  	balancerCC0 := val.(balancer.ClientConn)
  1691  
  1692  	// Set p0 to transient failure, p1 will be started.
  1693  	addrs0 := <-cc.NewSubConnAddrsCh
  1694  	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
  1695  		t.Fatalf("sc is created with addr %v, want %v", got, want)
  1696  	}
  1697  	sc0 := <-cc.NewSubConnCh
  1698  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
  1699  
  1700  	// Retrieve the ClientConn passed to the child policy from p1.
  1701  	val, err = ccCh.Receive(ctx)
  1702  	if err != nil {
  1703  		t.Fatalf("timeout waiting for ClientConn from the child policy")
  1704  	}
  1705  	balancerCC1 := val.(balancer.ClientConn)
  1706  
  1707  	// Since IgnoreReresolutionRequests was set to true for p0, ResolveNow()
  1708  	// from p0 should all be ignored.
  1709  	for i := 0; i < 5; i++ {
  1710  		balancerCC0.ResolveNow(resolver.ResolveNowOptions{})
  1711  	}
  1712  	select {
  1713  	case <-cc.ResolveNowCh:
  1714  		t.Fatalf("got unexpected ResolveNow() call")
  1715  	case <-time.After(defaultTestShortTimeout):
  1716  	}
  1717  
  1718  	// But IgnoreReresolutionRequests was false for p1, ResolveNow() from p1
  1719  	// should be forwarded.
  1720  	balancerCC1.ResolveNow(resolver.ResolveNowOptions{})
  1721  	select {
  1722  	case <-cc.ResolveNowCh:
  1723  	case <-time.After(defaultTestShortTimeout):
  1724  		t.Fatalf("timeout waiting for ResolveNow()")
  1725  	}
  1726  }
  1727  
  1728  const initIdleBalancerName = "test-init-Idle-balancer"
  1729  
  1730  var errsTestInitIdle = []error{
  1731  	fmt.Errorf("init Idle balancer error 0"),
  1732  	fmt.Errorf("init Idle balancer error 1"),
  1733  }
  1734  
  1735  func init() {
  1736  	for i := 0; i < 2; i++ {
  1737  		ii := i
  1738  		stub.Register(fmt.Sprintf("%s-%d", initIdleBalancerName, ii), stub.BalancerFuncs{
  1739  			UpdateClientConnState: func(bd *stub.BalancerData, opts balancer.ClientConnState) error {
  1740  				lis := func(state balancer.SubConnState) {
  1741  					err := fmt.Errorf("wrong picker error")
  1742  					if state.ConnectivityState == connectivity.Idle {
  1743  						err = errsTestInitIdle[ii]
  1744  					}
  1745  					bd.ClientConn.UpdateState(balancer.State{
  1746  						ConnectivityState: state.ConnectivityState,
  1747  						Picker:            &testutils.TestConstPicker{Err: err},
  1748  					})
  1749  				}
  1750  
  1751  				sc, err := bd.ClientConn.NewSubConn(opts.ResolverState.Addresses, balancer.NewSubConnOptions{StateListener: lis})
  1752  				if err != nil {
  1753  					return err
  1754  				}
  1755  				sc.Connect()
  1756  				bd.ClientConn.UpdateState(balancer.State{
  1757  					ConnectivityState: connectivity.Connecting,
  1758  					Picker:            &testutils.TestConstPicker{Err: balancer.ErrNoSubConnAvailable},
  1759  				})
  1760  				return nil
  1761  			},
  1762  		})
  1763  	}
  1764  }
  1765  
  1766  // If the high priorities send initial pickers with Idle state, their pickers
  1767  // should get picks, because policies like ringhash starts in Idle, and doesn't
  1768  // connect.
  1769  //
  1770  // Init 0, 1; 0 is Idle, use 0; 0 is down, start 1; 1 is Idle, use 1.
  1771  func (s) TestPriority_HighPriorityInitIdle(t *testing.T) {
  1772  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
  1773  	defer cancel()
  1774  
  1775  	cc := testutils.NewBalancerClientConn(t)
  1776  	bb := balancer.Get(Name)
  1777  	pb := bb.Build(cc, balancer.BuildOptions{})
  1778  	defer pb.Close()
  1779  
  1780  	// Two children, with priorities [0, 1], each with one backend.
  1781  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1782  		ResolverState: resolver.State{
  1783  			Addresses: []resolver.Address{
  1784  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1785  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
  1786  			},
  1787  		},
  1788  		BalancerConfig: &LBConfig{
  1789  			Children: map[string]*Child{
  1790  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: fmt.Sprintf("%s-%d", initIdleBalancerName, 0)}},
  1791  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: fmt.Sprintf("%s-%d", initIdleBalancerName, 1)}},
  1792  			},
  1793  			Priorities: []string{"child-0", "child-1"},
  1794  		},
  1795  	}); err != nil {
  1796  		t.Fatalf("failed to update ClientConn state: %v", err)
  1797  	}
  1798  
  1799  	addrs0 := <-cc.NewSubConnAddrsCh
  1800  	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
  1801  		t.Fatalf("sc is created with addr %v, want %v", got, want)
  1802  	}
  1803  	sc0 := <-cc.NewSubConnCh
  1804  
  1805  	// Send an Idle state update to trigger an Idle picker update.
  1806  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle})
  1807  	if err := cc.WaitForPickerWithErr(ctx, errsTestInitIdle[0]); err != nil {
  1808  		t.Fatal(err.Error())
  1809  	}
  1810  
  1811  	// Turn p0 down, to start p1.
  1812  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
  1813  	// Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs
  1814  	// will retry.
  1815  	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
  1816  		t.Fatal(err.Error())
  1817  	}
  1818  
  1819  	addrs1 := <-cc.NewSubConnAddrsCh
  1820  	if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want {
  1821  		t.Fatalf("sc is created with addr %v, want %v", got, want)
  1822  	}
  1823  	sc1 := <-cc.NewSubConnCh
  1824  	// Idle picker from p1 should also be forwarded.
  1825  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle})
  1826  	if err := cc.WaitForPickerWithErr(ctx, errsTestInitIdle[1]); err != nil {
  1827  		t.Fatal(err.Error())
  1828  	}
  1829  }
  1830  
  1831  // If the high priorities send initial pickers with Idle state, their pickers
  1832  // should get picks, because policies like ringhash starts in Idle, and doesn't
  1833  // connect. In this case, if a lower priority is added, it shouldn't switch to
  1834  // the lower priority.
  1835  //
  1836  // Init 0; 0 is Idle, use 0; add 1, use 0.
  1837  func (s) TestPriority_AddLowPriorityWhenHighIsInIdle(t *testing.T) {
  1838  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
  1839  	defer cancel()
  1840  
  1841  	cc := testutils.NewBalancerClientConn(t)
  1842  	bb := balancer.Get(Name)
  1843  	pb := bb.Build(cc, balancer.BuildOptions{})
  1844  	defer pb.Close()
  1845  
  1846  	// One child, with priorities [0], one backend.
  1847  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1848  		ResolverState: resolver.State{
  1849  			Addresses: []resolver.Address{
  1850  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1851  			},
  1852  		},
  1853  		BalancerConfig: &LBConfig{
  1854  			Children: map[string]*Child{
  1855  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: fmt.Sprintf("%s-%d", initIdleBalancerName, 0)}},
  1856  			},
  1857  			Priorities: []string{"child-0"},
  1858  		},
  1859  	}); err != nil {
  1860  		t.Fatalf("failed to update ClientConn state: %v", err)
  1861  	}
  1862  
  1863  	addrs0 := <-cc.NewSubConnAddrsCh
  1864  	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
  1865  		t.Fatalf("sc is created with addr %v, want %v", got, want)
  1866  	}
  1867  	sc0 := <-cc.NewSubConnCh
  1868  
  1869  	// Send an Idle state update to trigger an Idle picker update.
  1870  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Idle})
  1871  	if err := cc.WaitForPickerWithErr(ctx, errsTestInitIdle[0]); err != nil {
  1872  		t.Fatal(err.Error())
  1873  	}
  1874  
  1875  	// Add 1, should keep using 0.
  1876  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1877  		ResolverState: resolver.State{
  1878  			Addresses: []resolver.Address{
  1879  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1880  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
  1881  			},
  1882  		},
  1883  		BalancerConfig: &LBConfig{
  1884  			Children: map[string]*Child{
  1885  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: fmt.Sprintf("%s-%d", initIdleBalancerName, 0)}},
  1886  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: fmt.Sprintf("%s-%d", initIdleBalancerName, 1)}},
  1887  			},
  1888  			Priorities: []string{"child-0", "child-1"},
  1889  		},
  1890  	}); err != nil {
  1891  		t.Fatalf("failed to update ClientConn state: %v", err)
  1892  	}
  1893  
  1894  	// The ClientConn state update triggers a priority switch, from p0 -> p0
  1895  	// (since p0 is still in use). Along with this the update, p0 also gets a
  1896  	// ClientConn state update, with the addresses, which didn't change in this
  1897  	// test (this update to the child is necessary in case the addresses are
  1898  	// different).
  1899  	//
  1900  	// The test child policy, initIdleBalancer, blindly calls NewSubConn with
  1901  	// all the addresses it receives, so this will trigger a NewSubConn with the
  1902  	// old p0 addresses. (Note that in a real balancer, like roundrobin, no new
  1903  	// SubConn will be created because the addresses didn't change).
  1904  	//
  1905  	// The check below makes sure that the addresses are still from p0, and not
  1906  	// from p1. This is good enough for the purpose of this test.
  1907  	addrsNew := <-cc.NewSubConnAddrsCh
  1908  	if got, want := addrsNew[0].Addr, testBackendAddrStrs[0]; got != want {
  1909  		// Fail if p1 is started and creates a SubConn.
  1910  		t.Fatalf("got unexpected call to NewSubConn with addr: %v, want %v", addrsNew, want)
  1911  	}
  1912  }
  1913  
  1914  // Lower priority is used when higher priority is not ready; higher priority
  1915  // still gets updates.
  1916  //
  1917  // Init 0 and 1; 0 is down, 1 is up, use 1; update 0; 0 is up, use 0
  1918  func (s) TestPriority_HighPriorityUpdatesWhenLowInUse(t *testing.T) {
  1919  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
  1920  	defer cancel()
  1921  
  1922  	cc := testutils.NewBalancerClientConn(t)
  1923  	bb := balancer.Get(Name)
  1924  	pb := bb.Build(cc, balancer.BuildOptions{})
  1925  	defer pb.Close()
  1926  
  1927  	t.Log("Two localities, with priorities [0, 1], each with one backend.")
  1928  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1929  		ResolverState: resolver.State{
  1930  			Addresses: []resolver.Address{
  1931  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[0]}, []string{"child-0"}),
  1932  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[1]}, []string{"child-1"}),
  1933  			},
  1934  		},
  1935  		BalancerConfig: &LBConfig{
  1936  			Children: map[string]*Child{
  1937  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1938  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1939  			},
  1940  			Priorities: []string{"child-0", "child-1"},
  1941  		},
  1942  	}); err != nil {
  1943  		t.Fatalf("failed to update ClientConn state: %v", err)
  1944  	}
  1945  
  1946  	addrs0 := <-cc.NewSubConnAddrsCh
  1947  	if got, want := addrs0[0].Addr, testBackendAddrStrs[0]; got != want {
  1948  		t.Fatalf("sc is created with addr %v, want %v", got, want)
  1949  	}
  1950  	sc0 := <-cc.NewSubConnCh
  1951  
  1952  	t.Log("Make p0 fail.")
  1953  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
  1954  	sc0.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.TransientFailure})
  1955  
  1956  	// Before 1 gets READY, picker should return NoSubConnAvailable, so RPCs
  1957  	// will retry.
  1958  	if err := cc.WaitForPickerWithErr(ctx, balancer.ErrNoSubConnAvailable); err != nil {
  1959  		t.Fatal(err.Error())
  1960  	}
  1961  
  1962  	t.Log("Make p1 ready.")
  1963  	addrs1 := <-cc.NewSubConnAddrsCh
  1964  	if got, want := addrs1[0].Addr, testBackendAddrStrs[1]; got != want {
  1965  		t.Fatalf("sc is created with addr %v, want %v", got, want)
  1966  	}
  1967  	sc1 := <-cc.NewSubConnCh
  1968  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
  1969  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
  1970  
  1971  	// Test pick with 1.
  1972  	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
  1973  		t.Fatal(err.Error())
  1974  	}
  1975  
  1976  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
  1977  	// Does not change the aggregate state, because round robin does not leave
  1978  	// TRANSIENT_FAILURE if a subconn goes CONNECTING.
  1979  	sc1.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
  1980  
  1981  	if err := cc.WaitForRoundRobinPicker(ctx, sc1); err != nil {
  1982  		t.Fatal(err.Error())
  1983  	}
  1984  
  1985  	t.Log("Change p0 to use new address.")
  1986  	if err := pb.UpdateClientConnState(balancer.ClientConnState{
  1987  		ResolverState: resolver.State{
  1988  			Addresses: []resolver.Address{
  1989  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[2]}, []string{"child-0"}),
  1990  				hierarchy.Set(resolver.Address{Addr: testBackendAddrStrs[3]}, []string{"child-1"}),
  1991  			},
  1992  		},
  1993  		BalancerConfig: &LBConfig{
  1994  			Children: map[string]*Child{
  1995  				"child-0": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1996  				"child-1": {Config: &internalserviceconfig.BalancerConfig{Name: roundrobin.Name}},
  1997  			},
  1998  			Priorities: []string{"child-0", "child-1"},
  1999  		},
  2000  	}); err != nil {
  2001  		t.Fatalf("failed to update ClientConn state: %v", err)
  2002  	}
  2003  
  2004  	// Two new subconns are created by the previous update; one by p0 and one
  2005  	// by p1.  They don't happen concurrently, but they could happen in any
  2006  	// order.
  2007  	t.Log("Make p0 and p1 both ready; p0 should be used.")
  2008  	var sc2, sc3 balancer.SubConn
  2009  	for i := 0; i < 2; i++ {
  2010  		addr := <-cc.NewSubConnAddrsCh
  2011  		sc := <-cc.NewSubConnCh
  2012  		switch addr[0].Addr {
  2013  		case testBackendAddrStrs[2]:
  2014  			sc2 = sc
  2015  		case testBackendAddrStrs[3]:
  2016  			sc3 = sc
  2017  		default:
  2018  			t.Fatalf("sc is created with addr %v, want %v or %v", addr[0].Addr, testBackendAddrStrs[2], testBackendAddrStrs[3])
  2019  		}
  2020  		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Connecting})
  2021  		sc.UpdateState(balancer.SubConnState{ConnectivityState: connectivity.Ready})
  2022  	}
  2023  	if sc2 == nil {
  2024  		t.Fatalf("sc not created with addr %v", testBackendAddrStrs[2])
  2025  	}
  2026  	if sc3 == nil {
  2027  		t.Fatalf("sc not created with addr %v", testBackendAddrStrs[3])
  2028  	}
  2029  
  2030  	// Test pick with 0.
  2031  	if err := cc.WaitForRoundRobinPicker(ctx, sc2); err != nil {
  2032  		t.Fatal(err.Error())
  2033  	}
  2034  }
  2035  

View as plain text