xds_server_integration_test.go

Documentation: google.golang.org/grpc/test/xds

     1  /*
     2   *
     3   * Copyright 2020 gRPC authors.
     4   *
     5   * Licensed under the Apache License, Version 2.0 (the "License");
     6   * you may not use this file except in compliance with the License.
     7   * You may obtain a copy of the License at
     8   *
     9   *     http://www.apache.org/licenses/LICENSE-2.0
    10   *
    11   * Unless required by applicable law or agreed to in writing, software
    12   * distributed under the License is distributed on an "AS IS" BASIS,
    13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14   * See the License for the specific language governing permissions and
    15   * limitations under the License.
    16   *
    17   */
    18  
    19  package xds_test
    20  
    21  import (
    22  	"context"
    23  	"fmt"
    24  	"io"
    25  	"net"
    26  	"strconv"
    27  	"testing"
    28  
    29  	"google.golang.org/grpc"
    30  	"google.golang.org/grpc/codes"
    31  	"google.golang.org/grpc/credentials/insecure"
    32  	xdscreds "google.golang.org/grpc/credentials/xds"
    33  	"google.golang.org/grpc/internal/testutils"
    34  	"google.golang.org/grpc/internal/testutils/xds/e2e"
    35  	"google.golang.org/grpc/status"
    36  	"google.golang.org/grpc/xds"
    37  
    38  	testgrpc "google.golang.org/grpc/interop/grpc_testing"
    39  	testpb "google.golang.org/grpc/interop/grpc_testing"
    40  )
    41  
    42  type testService struct {
    43  	testgrpc.TestServiceServer
    44  }
    45  
    46  func (*testService) EmptyCall(context.Context, *testpb.Empty) (*testpb.Empty, error) {
    47  	return &testpb.Empty{}, nil
    48  }
    49  
    50  func (*testService) UnaryCall(context.Context, *testpb.SimpleRequest) (*testpb.SimpleResponse, error) {
    51  	return &testpb.SimpleResponse{}, nil
    52  }
    53  
    54  func (*testService) FullDuplexCall(stream testgrpc.TestService_FullDuplexCallServer) error {
    55  	for {
    56  		_, err := stream.Recv() // hangs here forever if stream doesn't shut down...doesn't receive EOF without any errors
    57  		if err == io.EOF {
    58  			return nil
    59  		}
    60  	}
    61  }
    62  
    63  func testModeChangeServerOption(t *testing.T) grpc.ServerOption {
    64  	// Create a server option to get notified about serving mode changes. We don't
    65  	// do anything other than throwing a log entry here. But this is required,
    66  	// since the server code emits a log entry at the default level (which is
    67  	// ERROR) if no callback is registered for serving mode changes. Our
    68  	// testLogger fails the test if there is any log entry at ERROR level. It does
    69  	// provide an ExpectError()  method, but that takes a string and it would be
    70  	// painful to construct the exact error message expected here. Instead this
    71  	// works just fine.
    72  	return xds.ServingModeCallback(func(addr net.Addr, args xds.ServingModeChangeArgs) {
    73  		t.Logf("Serving mode for listener %q changed to %q, err: %v", addr.String(), args.Mode, args.Err)
    74  	})
    75  }
    76  
    77  // setupGRPCServer performs the following:
    78  //   - spin up an xDS-enabled gRPC server, configure it with xdsCredentials and
    79  //     register the test service on it
    80  //   - create a local TCP listener and start serving on it
    81  //
    82  // Returns the following:
    83  // - local listener on which the xDS-enabled gRPC server is serving on
    84  // - cleanup function to be invoked by the tests when done
    85  func setupGRPCServer(t *testing.T, bootstrapContents []byte) (net.Listener, func()) {
    86  	t.Helper()
    87  
    88  	// Configure xDS credentials to be used on the server-side.
    89  	creds, err := xdscreds.NewServerCredentials(xdscreds.ServerOptions{
    90  		FallbackCreds: insecure.NewCredentials(),
    91  	})
    92  	if err != nil {
    93  		t.Fatal(err)
    94  	}
    95  
    96  	// Initialize an xDS-enabled gRPC server and register the stubServer on it.
    97  	server, err := xds.NewGRPCServer(grpc.Creds(creds), testModeChangeServerOption(t), xds.BootstrapContentsForTesting(bootstrapContents))
    98  	if err != nil {
    99  		t.Fatalf("Failed to create an xDS enabled gRPC server: %v", err)
   100  	}
   101  	testgrpc.RegisterTestServiceServer(server, &testService{})
   102  
   103  	// Create a local listener and pass it to Serve().
   104  	lis, err := testutils.LocalTCPListener()
   105  	if err != nil {
   106  		t.Fatalf("testutils.LocalTCPListener() failed: %v", err)
   107  	}
   108  
   109  	go func() {
   110  		if err := server.Serve(lis); err != nil {
   111  			t.Errorf("Serve() failed: %v", err)
   112  		}
   113  	}()
   114  
   115  	return lis, func() {
   116  		server.Stop()
   117  	}
   118  }
   119  
   120  func hostPortFromListener(lis net.Listener) (string, uint32, error) {
   121  	host, p, err := net.SplitHostPort(lis.Addr().String())
   122  	if err != nil {
   123  		return "", 0, fmt.Errorf("net.SplitHostPort(%s) failed: %v", lis.Addr().String(), err)
   124  	}
   125  	port, err := strconv.ParseInt(p, 10, 32)
   126  	if err != nil {
   127  		return "", 0, fmt.Errorf("strconv.ParseInt(%s, 10, 32) failed: %v", p, err)
   128  	}
   129  	return host, uint32(port), nil
   130  }
   131  
   132  // TestServerSideXDS_Fallback is an e2e test which verifies xDS credentials
   133  // fallback functionality.
   134  //
   135  // The following sequence of events happen as part of this test:
   136  //   - An xDS-enabled gRPC server is created and xDS credentials are configured.
   137  //   - xDS is enabled on the client by the use of the xds:/// scheme, and xDS
   138  //     credentials are configured.
   139  //   - Control plane is configured to not send any security configuration to both
   140  //     the client and the server. This results in both of them using the
   141  //     configured fallback credentials (which is insecure creds in this case).
   142  func (s) TestServerSideXDS_Fallback(t *testing.T) {
   143  	managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{})
   144  	defer cleanup1()
   145  
   146  	lis, cleanup2 := setupGRPCServer(t, bootstrapContents)
   147  	defer cleanup2()
   148  
   149  	// Grab the host and port of the server and create client side xDS resources
   150  	// corresponding to it. This contains default resources with no security
   151  	// configuration in the Cluster resources.
   152  	host, port, err := hostPortFromListener(lis)
   153  	if err != nil {
   154  		t.Fatalf("failed to retrieve host and port of server: %v", err)
   155  	}
   156  	const serviceName = "my-service-fallback"
   157  	resources := e2e.DefaultClientResources(e2e.ResourceParams{
   158  		DialTarget: serviceName,
   159  		NodeID:     nodeID,
   160  		Host:       host,
   161  		Port:       port,
   162  		SecLevel:   e2e.SecurityLevelNone,
   163  	})
   164  
   165  	// Create an inbound xDS listener resource for the server side that does not
   166  	// contain any security configuration. This should force the server-side
   167  	// xdsCredentials to use fallback.
   168  	inboundLis := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName")
   169  	resources.Listeners = append(resources.Listeners, inboundLis)
   170  
   171  	// Setup the management server with client and server-side resources.
   172  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
   173  	defer cancel()
   174  	if err := managementServer.Update(ctx, resources); err != nil {
   175  		t.Fatal(err)
   176  	}
   177  
   178  	// Create client-side xDS credentials with an insecure fallback.
   179  	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{
   180  		FallbackCreds: insecure.NewCredentials(),
   181  	})
   182  	if err != nil {
   183  		t.Fatal(err)
   184  	}
   185  
   186  	// Create a ClientConn with the xds scheme and make a successful RPC.
   187  	cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolver))
   188  	if err != nil {
   189  		t.Fatalf("failed to dial local test server: %v", err)
   190  	}
   191  	defer cc.Close()
   192  
   193  	client := testgrpc.NewTestServiceClient(cc)
   194  	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
   195  		t.Errorf("rpc EmptyCall() failed: %v", err)
   196  	}
   197  }
   198  
   199  // TestServerSideXDS_FileWatcherCerts is an e2e test which verifies xDS
   200  // credentials with file watcher certificate provider.
   201  //
   202  // The following sequence of events happen as part of this test:
   203  //   - An xDS-enabled gRPC server is created and xDS credentials are configured.
   204  //   - xDS is enabled on the client by the use of the xds:/// scheme, and xDS
   205  //     credentials are configured.
   206  //   - Control plane is configured to send security configuration to both the
   207  //     client and the server, pointing to the file watcher certificate provider.
   208  //     We verify both TLS and mTLS scenarios.
   209  func (s) TestServerSideXDS_FileWatcherCerts(t *testing.T) {
   210  	tests := []struct {
   211  		name     string
   212  		secLevel e2e.SecurityLevel
   213  	}{
   214  		{
   215  			name:     "tls",
   216  			secLevel: e2e.SecurityLevelTLS,
   217  		},
   218  		{
   219  			name:     "mtls",
   220  			secLevel: e2e.SecurityLevelMTLS,
   221  		},
   222  	}
   223  	for _, test := range tests {
   224  		t.Run(test.name, func(t *testing.T) {
   225  			managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{})
   226  			defer cleanup1()
   227  
   228  			lis, cleanup2 := setupGRPCServer(t, bootstrapContents)
   229  			defer cleanup2()
   230  
   231  			// Grab the host and port of the server and create client side xDS
   232  			// resources corresponding to it.
   233  			host, port, err := hostPortFromListener(lis)
   234  			if err != nil {
   235  				t.Fatalf("failed to retrieve host and port of server: %v", err)
   236  			}
   237  
   238  			// Create xDS resources to be consumed on the client side. This
   239  			// includes the listener, route configuration, cluster (with
   240  			// security configuration) and endpoint resources.
   241  			serviceName := "my-service-file-watcher-certs-" + test.name
   242  			resources := e2e.DefaultClientResources(e2e.ResourceParams{
   243  				DialTarget: serviceName,
   244  				NodeID:     nodeID,
   245  				Host:       host,
   246  				Port:       port,
   247  				SecLevel:   test.secLevel,
   248  			})
   249  
   250  			// Create an inbound xDS listener resource for the server side that
   251  			// contains security configuration pointing to the file watcher
   252  			// plugin.
   253  			inboundLis := e2e.DefaultServerListener(host, port, test.secLevel, "routeName")
   254  			resources.Listeners = append(resources.Listeners, inboundLis)
   255  
   256  			// Setup the management server with client and server resources.
   257  			ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
   258  			defer cancel()
   259  			if err := managementServer.Update(ctx, resources); err != nil {
   260  				t.Fatal(err)
   261  			}
   262  
   263  			// Create client-side xDS credentials with an insecure fallback.
   264  			creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{
   265  				FallbackCreds: insecure.NewCredentials(),
   266  			})
   267  			if err != nil {
   268  				t.Fatal(err)
   269  			}
   270  
   271  			// Create a ClientConn with the xds scheme and make an RPC.
   272  			cc, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolver))
   273  			if err != nil {
   274  				t.Fatalf("failed to dial local test server: %v", err)
   275  			}
   276  			defer cc.Close()
   277  
   278  			client := testgrpc.NewTestServiceClient(cc)
   279  			if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
   280  				t.Fatalf("rpc EmptyCall() failed: %v", err)
   281  			}
   282  		})
   283  	}
   284  }
   285  
   286  // TestServerSideXDS_SecurityConfigChange is an e2e test where xDS is enabled on
   287  // the server-side and xdsCredentials are configured for security. The control
   288  // plane initially does not any security configuration. This forces the
   289  // xdsCredentials to use fallback creds, which is this case is insecure creds.
   290  // We verify that a client connecting with TLS creds is not able to successfully
   291  // make an RPC. The control plane then sends a listener resource with security
   292  // configuration pointing to the use of the file_watcher plugin and we verify
   293  // that the same client is now able to successfully make an RPC.
   294  func (s) TestServerSideXDS_SecurityConfigChange(t *testing.T) {
   295  	managementServer, nodeID, bootstrapContents, resolver, cleanup1 := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{})
   296  	defer cleanup1()
   297  
   298  	lis, cleanup2 := setupGRPCServer(t, bootstrapContents)
   299  	defer cleanup2()
   300  
   301  	// Grab the host and port of the server and create client side xDS resources
   302  	// corresponding to it. This contains default resources with no security
   303  	// configuration in the Cluster resource. This should force the xDS
   304  	// credentials on the client to use its fallback.
   305  	host, port, err := hostPortFromListener(lis)
   306  	if err != nil {
   307  		t.Fatalf("failed to retrieve host and port of server: %v", err)
   308  	}
   309  	const serviceName = "my-service-security-config-change"
   310  	resources := e2e.DefaultClientResources(e2e.ResourceParams{
   311  		DialTarget: serviceName,
   312  		NodeID:     nodeID,
   313  		Host:       host,
   314  		Port:       port,
   315  		SecLevel:   e2e.SecurityLevelNone,
   316  	})
   317  
   318  	// Create an inbound xDS listener resource for the server side that does not
   319  	// contain any security configuration. This should force the xDS credentials
   320  	// on server to use its fallback.
   321  	inboundLis := e2e.DefaultServerListener(host, port, e2e.SecurityLevelNone, "routeName")
   322  	resources.Listeners = append(resources.Listeners, inboundLis)
   323  
   324  	// Setup the management server with client and server-side resources.
   325  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
   326  	defer cancel()
   327  	if err := managementServer.Update(ctx, resources); err != nil {
   328  		t.Fatal(err)
   329  	}
   330  
   331  	// Create client-side xDS credentials with an insecure fallback.
   332  	xdsCreds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{
   333  		FallbackCreds: insecure.NewCredentials(),
   334  	})
   335  	if err != nil {
   336  		t.Fatal(err)
   337  	}
   338  
   339  	// Create a ClientConn with the xds scheme and make a successful RPC.
   340  	xdsCC, err := grpc.DialContext(ctx, fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(xdsCreds), grpc.WithResolvers(resolver))
   341  	if err != nil {
   342  		t.Fatalf("failed to dial local test server: %v", err)
   343  	}
   344  	defer xdsCC.Close()
   345  
   346  	client := testgrpc.NewTestServiceClient(xdsCC)
   347  	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
   348  		t.Fatalf("rpc EmptyCall() failed: %v", err)
   349  	}
   350  
   351  	// Create a ClientConn with TLS creds. This should fail since the server is
   352  	// using fallback credentials which in this case in insecure creds.
   353  	tlsCreds := e2e.CreateClientTLSCredentials(t)
   354  	tlsCC, err := grpc.DialContext(ctx, lis.Addr().String(), grpc.WithTransportCredentials(tlsCreds))
   355  	if err != nil {
   356  		t.Fatalf("failed to dial local test server: %v", err)
   357  	}
   358  	defer tlsCC.Close()
   359  
   360  	// We don't set 'waitForReady` here since we want this call to failfast.
   361  	client = testgrpc.NewTestServiceClient(tlsCC)
   362  	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable {
   363  		t.Fatal("rpc EmptyCall() succeeded when expected to fail")
   364  	}
   365  
   366  	// Switch server and client side resources with ones that contain required
   367  	// security configuration for mTLS with a file watcher certificate provider.
   368  	resources = e2e.DefaultClientResources(e2e.ResourceParams{
   369  		DialTarget: serviceName,
   370  		NodeID:     nodeID,
   371  		Host:       host,
   372  		Port:       port,
   373  		SecLevel:   e2e.SecurityLevelMTLS,
   374  	})
   375  	inboundLis = e2e.DefaultServerListener(host, port, e2e.SecurityLevelMTLS, "routeName")
   376  	resources.Listeners = append(resources.Listeners, inboundLis)
   377  	if err := managementServer.Update(ctx, resources); err != nil {
   378  		t.Fatal(err)
   379  	}
   380  
   381  	// Make another RPC with `waitForReady` set and expect this to succeed.
   382  	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
   383  		t.Fatalf("rpc EmptyCall() failed: %v", err)
   384  	}
   385  }
   386
View as plain text