updater_test.go

Documentation: github.com/letsencrypt/boulder/crl/updater

     1  package updater
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"io"
     7  	"testing"
     8  	"time"
     9  
    10  	"google.golang.org/grpc"
    11  	"google.golang.org/protobuf/types/known/emptypb"
    12  	"google.golang.org/protobuf/types/known/timestamppb"
    13  
    14  	"github.com/jmhodges/clock"
    15  	capb "github.com/letsencrypt/boulder/ca/proto"
    16  	corepb "github.com/letsencrypt/boulder/core/proto"
    17  	cspb "github.com/letsencrypt/boulder/crl/storer/proto"
    18  	"github.com/letsencrypt/boulder/issuance"
    19  	blog "github.com/letsencrypt/boulder/log"
    20  	"github.com/letsencrypt/boulder/metrics"
    21  	"github.com/letsencrypt/boulder/mocks"
    22  	sapb "github.com/letsencrypt/boulder/sa/proto"
    23  	"github.com/letsencrypt/boulder/test"
    24  	"github.com/prometheus/client_golang/prometheus"
    25  )
    26  
    27  // fakeGRCC is a fake sapb.StorageAuthority_GetRevokedCertsClient which can be
    28  // populated with some CRL entries or an error for use as the return value of
    29  // a faked GetRevokedCerts call.
    30  type fakeGRCC struct {
    31  	grpc.ClientStream
    32  	entries []*corepb.CRLEntry
    33  	nextIdx int
    34  	err     error
    35  }
    36  
    37  func (f *fakeGRCC) Recv() (*corepb.CRLEntry, error) {
    38  	if f.err != nil {
    39  		return nil, f.err
    40  	}
    41  	if f.nextIdx < len(f.entries) {
    42  		res := f.entries[f.nextIdx]
    43  		f.nextIdx++
    44  		return res, nil
    45  	}
    46  	return nil, io.EOF
    47  }
    48  
    49  // fakeSAC is a fake sapb.StorageAuthorityClient which can be populated with a
    50  // fakeGRCC to be used as the return value for calls to GetRevokedCerts, and a
    51  // fake timestamp to serve as the database's maximum notAfter value.
    52  type fakeSAC struct {
    53  	mocks.StorageAuthority
    54  	grcc        fakeGRCC
    55  	maxNotAfter time.Time
    56  	leaseError  error
    57  }
    58  
    59  func (f *fakeSAC) GetRevokedCerts(ctx context.Context, _ *sapb.GetRevokedCertsRequest, _ ...grpc.CallOption) (sapb.StorageAuthority_GetRevokedCertsClient, error) {
    60  	return &f.grcc, nil
    61  }
    62  
    63  func (f *fakeSAC) GetMaxExpiration(_ context.Context, req *emptypb.Empty, _ ...grpc.CallOption) (*timestamppb.Timestamp, error) {
    64  	return timestamppb.New(f.maxNotAfter), nil
    65  }
    66  
    67  func (f *fakeSAC) LeaseCRLShard(_ context.Context, req *sapb.LeaseCRLShardRequest, _ ...grpc.CallOption) (*sapb.LeaseCRLShardResponse, error) {
    68  	if f.leaseError != nil {
    69  		return nil, f.leaseError
    70  	}
    71  	return &sapb.LeaseCRLShardResponse{IssuerNameID: req.IssuerNameID, ShardIdx: req.MinShardIdx}, nil
    72  }
    73  
    74  // fakeGCC is a fake capb.CRLGenerator_GenerateCRLClient which can be
    75  // populated with some CRL entries or an error for use as the return value of
    76  // a faked GenerateCRL call.
    77  type fakeGCC struct {
    78  	grpc.ClientStream
    79  	chunks  [][]byte
    80  	nextIdx int
    81  	sendErr error
    82  	recvErr error
    83  }
    84  
    85  func (f *fakeGCC) Send(*capb.GenerateCRLRequest) error {
    86  	return f.sendErr
    87  }
    88  
    89  func (f *fakeGCC) CloseSend() error {
    90  	return nil
    91  }
    92  
    93  func (f *fakeGCC) Recv() (*capb.GenerateCRLResponse, error) {
    94  	if f.recvErr != nil {
    95  		return nil, f.recvErr
    96  	}
    97  	if f.nextIdx < len(f.chunks) {
    98  		res := f.chunks[f.nextIdx]
    99  		f.nextIdx++
   100  		return &capb.GenerateCRLResponse{Chunk: res}, nil
   101  	}
   102  	return nil, io.EOF
   103  }
   104  
   105  // fakeCGC is a fake capb.CRLGeneratorClient which can be populated with a
   106  // fakeGCC to be used as the return value for calls to GenerateCRL.
   107  type fakeCGC struct {
   108  	gcc fakeGCC
   109  }
   110  
   111  func (f *fakeCGC) GenerateCRL(ctx context.Context, opts ...grpc.CallOption) (capb.CRLGenerator_GenerateCRLClient, error) {
   112  	return &f.gcc, nil
   113  }
   114  
   115  // fakeUCC is a fake cspb.CRLStorer_UploadCRLClient which can be populated with
   116  // an error for use as the return value of a faked UploadCRL call.
   117  type fakeUCC struct {
   118  	grpc.ClientStream
   119  	sendErr error
   120  	recvErr error
   121  }
   122  
   123  func (f *fakeUCC) Send(*cspb.UploadCRLRequest) error {
   124  	return f.sendErr
   125  }
   126  
   127  func (f *fakeUCC) CloseAndRecv() (*emptypb.Empty, error) {
   128  	if f.recvErr != nil {
   129  		return nil, f.recvErr
   130  	}
   131  	return &emptypb.Empty{}, nil
   132  }
   133  
   134  // fakeCSC is a fake cspb.CRLStorerClient which can be populated with a
   135  // fakeUCC for use as the return value for calls to UploadCRL.
   136  type fakeCSC struct {
   137  	ucc fakeUCC
   138  }
   139  
   140  func (f *fakeCSC) UploadCRL(ctx context.Context, opts ...grpc.CallOption) (cspb.CRLStorer_UploadCRLClient, error) {
   141  	return &f.ucc, nil
   142  }
   143  
   144  func TestUpdateShard(t *testing.T) {
   145  	e1, err := issuance.LoadCertificate("../../test/hierarchy/int-e1.cert.pem")
   146  	test.AssertNotError(t, err, "loading test issuer")
   147  	r3, err := issuance.LoadCertificate("../../test/hierarchy/int-r3.cert.pem")
   148  	test.AssertNotError(t, err, "loading test issuer")
   149  
   150  	sentinelErr := errors.New("oops")
   151  	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
   152  	defer cancel()
   153  
   154  	clk := clock.NewFake()
   155  	clk.Set(time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC))
   156  	cu, err := NewUpdater(
   157  		[]*issuance.Certificate{e1, r3},
   158  		2, 18*time.Hour, 24*time.Hour,
   159  		6*time.Hour, time.Minute, 1, 1,
   160  		&fakeSAC{grcc: fakeGRCC{}, maxNotAfter: clk.Now().Add(90 * 24 * time.Hour)},
   161  		&fakeCGC{gcc: fakeGCC{}},
   162  		&fakeCSC{ucc: fakeUCC{}},
   163  		metrics.NoopRegisterer, blog.NewMock(), clk,
   164  	)
   165  	test.AssertNotError(t, err, "building test crlUpdater")
   166  
   167  	testChunks := []chunk{
   168  		{clk.Now(), clk.Now().Add(18 * time.Hour), 0},
   169  	}
   170  
   171  	// Ensure that getting no results from the SA still works.
   172  	err = cu.updateShard(ctx, cu.clk.Now(), e1.NameID(), 0, testChunks)
   173  	test.AssertNotError(t, err, "empty CRL")
   174  	test.AssertMetricWithLabelsEquals(t, cu.updatedCounter, prometheus.Labels{
   175  		"issuer": "(TEST) Elegant Elephant E1", "result": "success",
   176  	}, 1)
   177  	cu.updatedCounter.Reset()
   178  
   179  	// Errors closing the Storer upload stream should bubble up.
   180  	cu.cs = &fakeCSC{ucc: fakeUCC{recvErr: sentinelErr}}
   181  	err = cu.updateShard(ctx, cu.clk.Now(), e1.NameID(), 0, testChunks)
   182  	test.AssertError(t, err, "storer error")
   183  	test.AssertContains(t, err.Error(), "closing CRLStorer upload stream")
   184  	test.AssertErrorIs(t, err, sentinelErr)
   185  	test.AssertMetricWithLabelsEquals(t, cu.updatedCounter, prometheus.Labels{
   186  		"issuer": "(TEST) Elegant Elephant E1", "result": "failed",
   187  	}, 1)
   188  	cu.updatedCounter.Reset()
   189  
   190  	// Errors sending to the Storer should bubble up sooner.
   191  	cu.cs = &fakeCSC{ucc: fakeUCC{sendErr: sentinelErr}}
   192  	err = cu.updateShard(ctx, cu.clk.Now(), e1.NameID(), 0, testChunks)
   193  	test.AssertError(t, err, "storer error")
   194  	test.AssertContains(t, err.Error(), "sending CRLStorer metadata")
   195  	test.AssertErrorIs(t, err, sentinelErr)
   196  	test.AssertMetricWithLabelsEquals(t, cu.updatedCounter, prometheus.Labels{
   197  		"issuer": "(TEST) Elegant Elephant E1", "result": "failed",
   198  	}, 1)
   199  	cu.updatedCounter.Reset()
   200  
   201  	// Errors reading from the CA should bubble up sooner.
   202  	cu.ca = &fakeCGC{gcc: fakeGCC{recvErr: sentinelErr}}
   203  	err = cu.updateShard(ctx, cu.clk.Now(), e1.NameID(), 0, testChunks)
   204  	test.AssertError(t, err, "CA error")
   205  	test.AssertContains(t, err.Error(), "receiving CRL bytes")
   206  	test.AssertErrorIs(t, err, sentinelErr)
   207  	test.AssertMetricWithLabelsEquals(t, cu.updatedCounter, prometheus.Labels{
   208  		"issuer": "(TEST) Elegant Elephant E1", "result": "failed",
   209  	}, 1)
   210  	cu.updatedCounter.Reset()
   211  
   212  	// Errors sending to the CA should bubble up sooner.
   213  	cu.ca = &fakeCGC{gcc: fakeGCC{sendErr: sentinelErr}}
   214  	err = cu.updateShard(ctx, cu.clk.Now(), e1.NameID(), 0, testChunks)
   215  	test.AssertError(t, err, "CA error")
   216  	test.AssertContains(t, err.Error(), "sending CA metadata")
   217  	test.AssertErrorIs(t, err, sentinelErr)
   218  	test.AssertMetricWithLabelsEquals(t, cu.updatedCounter, prometheus.Labels{
   219  		"issuer": "(TEST) Elegant Elephant E1", "result": "failed",
   220  	}, 1)
   221  	cu.updatedCounter.Reset()
   222  
   223  	// Errors reading from the SA should bubble up soonest.
   224  	cu.sa = &fakeSAC{grcc: fakeGRCC{err: sentinelErr}, maxNotAfter: clk.Now().Add(90 * 24 * time.Hour)}
   225  	err = cu.updateShard(ctx, cu.clk.Now(), e1.NameID(), 0, testChunks)
   226  	test.AssertError(t, err, "database error")
   227  	test.AssertContains(t, err.Error(), "retrieving entry from SA")
   228  	test.AssertErrorIs(t, err, sentinelErr)
   229  	test.AssertMetricWithLabelsEquals(t, cu.updatedCounter, prometheus.Labels{
   230  		"issuer": "(TEST) Elegant Elephant E1", "result": "failed",
   231  	}, 1)
   232  	cu.updatedCounter.Reset()
   233  }
   234  
   235  func TestUpdateShardWithRetry(t *testing.T) {
   236  	e1, err := issuance.LoadCertificate("../../test/hierarchy/int-e1.cert.pem")
   237  	test.AssertNotError(t, err, "loading test issuer")
   238  	r3, err := issuance.LoadCertificate("../../test/hierarchy/int-r3.cert.pem")
   239  	test.AssertNotError(t, err, "loading test issuer")
   240  
   241  	sentinelErr := errors.New("oops")
   242  	ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
   243  	defer cancel()
   244  
   245  	clk := clock.NewFake()
   246  	clk.Set(time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC))
   247  
   248  	// Build an updater that will always fail when it talks to the SA.
   249  	cu, err := NewUpdater(
   250  		[]*issuance.Certificate{e1, r3},
   251  		2, 18*time.Hour, 24*time.Hour,
   252  		6*time.Hour, time.Minute, 1, 1,
   253  		&fakeSAC{grcc: fakeGRCC{err: sentinelErr}, maxNotAfter: clk.Now().Add(90 * 24 * time.Hour)},
   254  		&fakeCGC{gcc: fakeGCC{}},
   255  		&fakeCSC{ucc: fakeUCC{}},
   256  		metrics.NoopRegisterer, blog.NewMock(), clk,
   257  	)
   258  	test.AssertNotError(t, err, "building test crlUpdater")
   259  
   260  	testChunks := []chunk{
   261  		{clk.Now(), clk.Now().Add(18 * time.Hour), 0},
   262  	}
   263  
   264  	// Ensure that having MaxAttempts set to 1 results in the clock not moving
   265  	// forward at all.
   266  	startTime := cu.clk.Now()
   267  	err = cu.updateShardWithRetry(ctx, cu.clk.Now(), e1.NameID(), 0, testChunks)
   268  	test.AssertError(t, err, "database error")
   269  	test.AssertErrorIs(t, err, sentinelErr)
   270  	test.AssertEquals(t, cu.clk.Now(), startTime)
   271  
   272  	// Ensure that having MaxAttempts set to 5 results in the clock moving forward
   273  	// by 1+2+4+8=15 seconds. The core.RetryBackoff system has 20% jitter built
   274  	// in, so we have to be approximate.
   275  	cu.maxAttempts = 5
   276  	startTime = cu.clk.Now()
   277  	err = cu.updateShardWithRetry(ctx, cu.clk.Now(), e1.NameID(), 0, testChunks)
   278  	test.AssertError(t, err, "database error")
   279  	test.AssertErrorIs(t, err, sentinelErr)
   280  	t.Logf("start: %v", startTime)
   281  	t.Logf("now: %v", cu.clk.Now())
   282  	test.Assert(t, startTime.Add(15*0.8*time.Second).Before(cu.clk.Now()), "retries didn't sleep enough")
   283  	test.Assert(t, startTime.Add(15*1.2*time.Second).After(cu.clk.Now()), "retries slept too much")
   284  }
   285
View as plain text