xds_client_certificate_providers_test.go

Documentation: google.golang.org/grpc/test/xds

     1  /*
     2   *
     3   * Copyright 2023 gRPC authors.
     4   *
     5   * Licensed under the Apache License, Version 2.0 (the "License");
     6   * you may not use this file except in compliance with the License.
     7   * You may obtain a copy of the License at
     8   *
     9   *     http://www.apache.org/licenses/LICENSE-2.0
    10   *
    11   * Unless required by applicable law or agreed to in writing, software
    12   * distributed under the License is distributed on an "AS IS" BASIS,
    13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14   * See the License for the specific language governing permissions and
    15   * limitations under the License.
    16   *
    17   */
    18  
    19  package xds_test
    20  
    21  import (
    22  	"context"
    23  	"crypto/tls"
    24  	"fmt"
    25  	"strings"
    26  	"testing"
    27  
    28  	"github.com/google/uuid"
    29  	"google.golang.org/grpc"
    30  	"google.golang.org/grpc/codes"
    31  	"google.golang.org/grpc/connectivity"
    32  	"google.golang.org/grpc/credentials/insecure"
    33  	xdscreds "google.golang.org/grpc/credentials/xds"
    34  	"google.golang.org/grpc/internal"
    35  	"google.golang.org/grpc/internal/stubserver"
    36  	"google.golang.org/grpc/internal/testutils"
    37  	"google.golang.org/grpc/internal/testutils/xds/bootstrap"
    38  	"google.golang.org/grpc/internal/testutils/xds/e2e"
    39  	"google.golang.org/grpc/peer"
    40  	"google.golang.org/grpc/resolver"
    41  	"google.golang.org/grpc/status"
    42  
    43  	v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
    44  	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
    45  	v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
    46  	v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
    47  	v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
    48  	v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
    49  	testgrpc "google.golang.org/grpc/interop/grpc_testing"
    50  	testpb "google.golang.org/grpc/interop/grpc_testing"
    51  )
    52  
    53  // Tests the case where the bootstrap configuration contains no certificate
    54  // providers, and xDS credentials with an insecure fallback is specified at dial
    55  // time. The management server is configured to return client side xDS resources
    56  // with no security configuration. The test verifies that the gRPC client is
    57  // able to make RPCs to the backend which is configured to accept plaintext
    58  // connections. This ensures that the insecure fallback credentials are getting
    59  // used on the client.
    60  func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Success(t *testing.T) {
    61  	// Spin up an xDS management server.
    62  	mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{})
    63  	if err != nil {
    64  		t.Fatalf("Failed to start management server: %v", err)
    65  	}
    66  	defer mgmtServer.Stop()
    67  
    68  	// Create bootstrap configuration with no certificate providers.
    69  	nodeID := uuid.New().String()
    70  	bs, err := bootstrap.Contents(bootstrap.Options{
    71  		NodeID:    nodeID,
    72  		ServerURI: mgmtServer.Address,
    73  	})
    74  	if err != nil {
    75  		t.Fatalf("Failed to create bootstrap configuration: %v", err)
    76  	}
    77  
    78  	// Create an xDS resolver with the above bootstrap configuration.
    79  	newResolver := internal.NewXDSResolverWithConfigForTesting
    80  	if newResolver == nil {
    81  		t.Fatal("internal.NewXDSResolverWithConfigForTesting is unset")
    82  	}
    83  	resolverBuilder, err := newResolver.(func([]byte) (resolver.Builder, error))(bs)
    84  	if err != nil {
    85  		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
    86  	}
    87  
    88  	// Spin up a test backend.
    89  	server := stubserver.StartTestService(t, nil)
    90  	defer server.Stop()
    91  
    92  	// Configure client side xDS resources on the management server, with no
    93  	// security configuration in the Cluster resource.
    94  	const serviceName = "my-service-client-side-xds"
    95  	resources := e2e.DefaultClientResources(e2e.ResourceParams{
    96  		DialTarget: serviceName,
    97  		NodeID:     nodeID,
    98  		Host:       "localhost",
    99  		Port:       testutils.ParsePort(t, server.Address),
   100  		SecLevel:   e2e.SecurityLevelNone,
   101  	})
   102  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
   103  	defer cancel()
   104  	if err := mgmtServer.Update(ctx, resources); err != nil {
   105  		t.Fatal(err)
   106  	}
   107  
   108  	// Create client-side xDS credentials with an insecure fallback.
   109  	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
   110  	if err != nil {
   111  		t.Fatal(err)
   112  	}
   113  
   114  	// Create a ClientConn and make a successful RPC.
   115  	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolverBuilder))
   116  	if err != nil {
   117  		t.Fatalf("failed to dial local test server: %v", err)
   118  	}
   119  	defer cc.Close()
   120  
   121  	client := testgrpc.NewTestServiceClient(cc)
   122  	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true)); err != nil {
   123  		t.Fatalf("EmptyCall() failed: %v", err)
   124  	}
   125  }
   126  
   127  // Tests the case where the bootstrap configuration contains no certificate
   128  // providers, and xDS credentials with an insecure fallback is specified at dial
   129  // time. The management server is configured to return client side xDS resources
   130  // with an mTLS security configuration. The test verifies that the gRPC client
   131  // moves to TRANSIENT_FAILURE and rpcs fail with the expected error code and
   132  // string. This ensures that when the certificate provider instance name
   133  // specified in the security configuration is not present in the bootstrap,
   134  // channel creation does not fail, but it moves to TRANSIENT_FAILURE and
   135  // subsequent rpcs fail.
   136  func (s) TestClientSideXDS_WithNoCertificateProvidersInBootstrap_Failure(t *testing.T) {
   137  	// Spin up an xDS management server.
   138  	mgmtServer, err := e2e.StartManagementServer(e2e.ManagementServerOptions{})
   139  	if err != nil {
   140  		t.Fatalf("Failed to start management server: %v", err)
   141  	}
   142  	defer mgmtServer.Stop()
   143  
   144  	// Create bootstrap configuration with no certificate providers.
   145  	nodeID := uuid.New().String()
   146  	bs, err := bootstrap.Contents(bootstrap.Options{
   147  		NodeID:    nodeID,
   148  		ServerURI: mgmtServer.Address,
   149  	})
   150  	if err != nil {
   151  		t.Fatalf("Failed to create bootstrap configuration: %v", err)
   152  	}
   153  
   154  	// Create an xDS resolver with the above bootstrap configuration.
   155  	newResolver := internal.NewXDSResolverWithConfigForTesting
   156  	if newResolver == nil {
   157  		t.Fatal("internal.NewXDSResolverWithConfigForTesting is unset")
   158  	}
   159  	resolverBuilder, err := newResolver.(func([]byte) (resolver.Builder, error))(bs)
   160  	if err != nil {
   161  		t.Fatalf("Failed to create xDS resolver for testing: %v", err)
   162  	}
   163  
   164  	// Spin up a test backend.
   165  	server := stubserver.StartTestService(t, nil)
   166  	defer server.Stop()
   167  
   168  	// Configure client side xDS resources on the management server, with mTLS
   169  	// security configuration in the Cluster resource.
   170  	const serviceName = "my-service-client-side-xds"
   171  	const clusterName = "cluster-" + serviceName
   172  	const endpointsName = "endpoints-" + serviceName
   173  	resources := e2e.DefaultClientResources(e2e.ResourceParams{
   174  		DialTarget: serviceName,
   175  		NodeID:     nodeID,
   176  		Host:       "localhost",
   177  		Port:       testutils.ParsePort(t, server.Address),
   178  		SecLevel:   e2e.SecurityLevelNone,
   179  	})
   180  	resources.Clusters = []*v3clusterpb.Cluster{e2e.DefaultCluster(clusterName, endpointsName, e2e.SecurityLevelMTLS)}
   181  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
   182  	defer cancel()
   183  	if err := mgmtServer.Update(ctx, resources); err != nil {
   184  		t.Fatal(err)
   185  	}
   186  
   187  	// Create client-side xDS credentials with an insecure fallback.
   188  	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
   189  	if err != nil {
   190  		t.Fatal(err)
   191  	}
   192  
   193  	// Create a ClientConn and ensure that it moves to TRANSIENT_FAILURE.
   194  	cc, err := grpc.Dial(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolverBuilder))
   195  	if err != nil {
   196  		t.Fatalf("failed to dial local test server: %v", err)
   197  	}
   198  	defer cc.Close()
   199  	testutils.AwaitState(ctx, t, cc, connectivity.TransientFailure)
   200  
   201  	// Make an RPC and ensure that expected error is returned.
   202  	wantErr := fmt.Sprintf("identity certificate provider instance name %q missing in bootstrap configuration", e2e.ClientSideCertProviderInstance)
   203  	client := testgrpc.NewTestServiceClient(cc)
   204  	if _, err := client.EmptyCall(ctx, &testpb.Empty{}); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
   205  		t.Fatalf("EmptyCall() failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
   206  	}
   207  }
   208  
   209  // Tests the case where the bootstrap configuration contains one certificate
   210  // provider, and xDS credentials with an insecure fallback is specified at dial
   211  // time. The management server responds with three clusters:
   212  //  1. contains valid security configuration pointing to the certificate provider
   213  //     instance specified in the bootstrap
   214  //  2. contains no security configuration, hence should use insecure fallback
   215  //  3. contains invalid security configuration pointing to a non-existent
   216  //     certificate provider instance
   217  //
   218  // The test verifies that RPCs to the first two clusters succeed, while RPCs to
   219  // the third cluster fails with an appropriate code and error message.
   220  func (s) TestClientSideXDS_WithValidAndInvalidSecurityConfiguration(t *testing.T) {
   221  	// Spin up an xDS management server. This uses a bootstrap config with a
   222  	// certificate provider instance name e2e.ClientSideCertProviderInstance.
   223  	mgmtServer, nodeID, _, resolver, cleanup := e2e.SetupManagementServer(t, e2e.ManagementServerOptions{AllowResourceSubset: true})
   224  	defer cleanup()
   225  
   226  	// Create test backends for all three clusters
   227  	// backend1 configured with TLS creds, represents cluster1
   228  	// backend2 configured with insecure creds, represents cluster2
   229  	// backend3 configured with insecure creds, represents cluster3
   230  	creds := e2e.CreateServerTLSCredentials(t, tls.RequireAndVerifyClientCert)
   231  	server1 := stubserver.StartTestService(t, nil, grpc.Creds(creds))
   232  	defer server1.Stop()
   233  	server2 := stubserver.StartTestService(t, nil)
   234  	defer server2.Stop()
   235  	server3 := stubserver.StartTestService(t, nil)
   236  	defer server3.Stop()
   237  
   238  	// Configure client side xDS resources on the management server.
   239  	const serviceName = "my-service-client-side-xds"
   240  	const routeConfigName = "route-" + serviceName
   241  	const clusterName1 = "cluster1-" + serviceName
   242  	const clusterName2 = "cluster2-" + serviceName
   243  	const clusterName3 = "cluster3-" + serviceName
   244  	const endpointsName1 = "endpoints1-" + serviceName
   245  	const endpointsName2 = "endpoints2-" + serviceName
   246  	const endpointsName3 = "endpoints3-" + serviceName
   247  	listeners := []*v3listenerpb.Listener{e2e.DefaultClientListener(serviceName, routeConfigName)}
   248  	// Route configuration:
   249  	// - "/grpc.testing.TestService/EmptyCall" --> cluster1
   250  	// - "/grpc.testing.TestService/UnaryCall" --> cluster2
   251  	// - "/grpc.testing.TestService/FullDuplexCall" --> cluster3
   252  	routes := []*v3routepb.RouteConfiguration{{
   253  		Name: routeConfigName,
   254  		VirtualHosts: []*v3routepb.VirtualHost{{
   255  			Domains: []string{serviceName},
   256  			Routes: []*v3routepb.Route{
   257  				{
   258  					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/EmptyCall"}},
   259  					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
   260  						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName1},
   261  					}},
   262  				},
   263  				{
   264  					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/UnaryCall"}},
   265  					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
   266  						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName2},
   267  					}},
   268  				},
   269  				{
   270  					Match: &v3routepb.RouteMatch{PathSpecifier: &v3routepb.RouteMatch_Prefix{Prefix: "/grpc.testing.TestService/FullDuplexCall"}},
   271  					Action: &v3routepb.Route_Route{Route: &v3routepb.RouteAction{
   272  						ClusterSpecifier: &v3routepb.RouteAction_Cluster{Cluster: clusterName3},
   273  					}},
   274  				},
   275  			},
   276  		}},
   277  	}}
   278  	// Clusters:
   279  	// - cluster1 with cert provider name e2e.ClientSideCertProviderInstance.
   280  	// - cluster2 with no security configuration.
   281  	// - cluster3 with non-existent cert provider name.
   282  	clusters := []*v3clusterpb.Cluster{
   283  		e2e.DefaultCluster(clusterName1, endpointsName1, e2e.SecurityLevelMTLS),
   284  		e2e.DefaultCluster(clusterName2, endpointsName2, e2e.SecurityLevelNone),
   285  		func() *v3clusterpb.Cluster {
   286  			cluster3 := e2e.DefaultCluster(clusterName3, endpointsName3, e2e.SecurityLevelMTLS)
   287  			cluster3.TransportSocket = &v3corepb.TransportSocket{
   288  				Name: "envoy.transport_sockets.tls",
   289  				ConfigType: &v3corepb.TransportSocket_TypedConfig{
   290  					TypedConfig: testutils.MarshalAny(t, &v3tlspb.UpstreamTlsContext{
   291  						CommonTlsContext: &v3tlspb.CommonTlsContext{
   292  							ValidationContextType: &v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance{
   293  								ValidationContextCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
   294  									InstanceName: "non-existent-certificate-provider-instance-name",
   295  								},
   296  							},
   297  							TlsCertificateCertificateProviderInstance: &v3tlspb.CommonTlsContext_CertificateProviderInstance{
   298  								InstanceName: "non-existent-certificate-provider-instance-name",
   299  							},
   300  						},
   301  					}),
   302  				},
   303  			}
   304  			return cluster3
   305  		}(),
   306  	}
   307  	// Endpoints for each of the above clusters with backends created earlier.
   308  	endpoints := []*v3endpointpb.ClusterLoadAssignment{
   309  		e2e.DefaultEndpoint(endpointsName1, "localhost", []uint32{testutils.ParsePort(t, server1.Address)}),
   310  		e2e.DefaultEndpoint(endpointsName2, "localhost", []uint32{testutils.ParsePort(t, server2.Address)}),
   311  	}
   312  	resources := e2e.UpdateOptions{
   313  		NodeID:         nodeID,
   314  		Listeners:      listeners,
   315  		Routes:         routes,
   316  		Clusters:       clusters,
   317  		Endpoints:      endpoints,
   318  		SkipValidation: true,
   319  	}
   320  	ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
   321  	defer cancel()
   322  	if err := mgmtServer.Update(ctx, resources); err != nil {
   323  		t.Fatal(err)
   324  	}
   325  
   326  	// Create client-side xDS credentials with an insecure fallback.
   327  	creds, err := xdscreds.NewClientCredentials(xdscreds.ClientOptions{FallbackCreds: insecure.NewCredentials()})
   328  	if err != nil {
   329  		t.Fatal(err)
   330  	}
   331  
   332  	// Create a ClientConn.
   333  	cc, err := grpc.NewClient(fmt.Sprintf("xds:///%s", serviceName), grpc.WithTransportCredentials(creds), grpc.WithResolvers(resolver))
   334  	if err != nil {
   335  		t.Fatalf("failed to dial local test server: %v", err)
   336  	}
   337  	defer cc.Close()
   338  
   339  	// Make an RPC to be routed to cluster1 and verify that it succeeds.
   340  	client := testgrpc.NewTestServiceClient(cc)
   341  	peer := &peer.Peer{}
   342  	if _, err := client.EmptyCall(ctx, &testpb.Empty{}, grpc.WaitForReady(true), grpc.Peer(peer)); err != nil {
   343  		t.Fatalf("EmptyCall() failed: %v", err)
   344  	}
   345  	if got, want := peer.Addr.String(), server1.Address; got != want {
   346  		t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
   347  
   348  	}
   349  
   350  	// Make an RPC to be routed to cluster2 and verify that it succeeds.
   351  	if _, err := client.UnaryCall(ctx, &testpb.SimpleRequest{}, grpc.Peer(peer)); err != nil {
   352  		t.Fatalf("UnaryCall() failed: %v", err)
   353  	}
   354  	if got, want := peer.Addr.String(), server2.Address; got != want {
   355  		t.Errorf("EmptyCall() routed to %q, want to be routed to: %q", got, want)
   356  	}
   357  
   358  	// Make an RPC to be routed to cluster3 and verify that it fails.
   359  	const wantErr = `identity certificate provider instance name "non-existent-certificate-provider-instance-name" missing in bootstrap configuration`
   360  	if _, err := client.FullDuplexCall(ctx); status.Code(err) != codes.Unavailable || !strings.Contains(err.Error(), wantErr) {
   361  		t.Fatalf("FullDuplexCall failed: %v, wantCode: %s, wantErr: %s", err, codes.Unavailable, wantErr)
   362  	}
   363  }
   364
View as plain text