Source file src/github.com/letsencrypt/boulder/va/va_test.go

Documentation: github.com/letsencrypt/boulder/va

     1  package va
     2  
     3  import (
     4  	"context"
     5  	"crypto/rsa"
     6  	"encoding/base64"
     7  	"errors"
     8  	"fmt"
     9  	"math/big"
    10  	"net"
    11  	"net/http"
    12  	"net/http/httptest"
    13  	"os"
    14  	"strings"
    15  	"sync"
    16  	"syscall"
    17  	"testing"
    18  	"time"
    19  
    20  	"github.com/jmhodges/clock"
    21  	"github.com/letsencrypt/boulder/bdns"
    22  	"github.com/letsencrypt/boulder/core"
    23  	corepb "github.com/letsencrypt/boulder/core/proto"
    24  	"github.com/letsencrypt/boulder/features"
    25  	"github.com/letsencrypt/boulder/identifier"
    26  	blog "github.com/letsencrypt/boulder/log"
    27  	"github.com/letsencrypt/boulder/metrics"
    28  	"github.com/letsencrypt/boulder/probs"
    29  	"github.com/letsencrypt/boulder/test"
    30  	vapb "github.com/letsencrypt/boulder/va/proto"
    31  	"github.com/prometheus/client_golang/prometheus"
    32  	"google.golang.org/grpc"
    33  	"gopkg.in/go-jose/go-jose.v2"
    34  )
    35  
    36  func TestImplementation(t *testing.T) {
    37  	test.AssertImplementsGRPCServer(t, &ValidationAuthorityImpl{}, vapb.UnimplementedVAServer{})
    38  	test.AssertImplementsGRPCServer(t, &ValidationAuthorityImpl{}, vapb.UnimplementedCAAServer{})
    39  }
    40  
    41  var expectedToken = "LoqXcYV8q5ONbJQxbmR7SCTNo3tiAXDfowyjxAjEuX0"
    42  var expectedKeyAuthorization = "LoqXcYV8q5ONbJQxbmR7SCTNo3tiAXDfowyjxAjEuX0.9jg46WB3rR_AHD-EBXdN7cBkH1WOu0tA3M9fm21mqTI"
    43  
    44  func bigIntFromB64(b64 string) *big.Int {
    45  	bytes, _ := base64.URLEncoding.DecodeString(b64)
    46  	x := big.NewInt(0)
    47  	x.SetBytes(bytes)
    48  	return x
    49  }
    50  
    51  func intFromB64(b64 string) int {
    52  	return int(bigIntFromB64(b64).Int64())
    53  }
    54  
    55  var n = bigIntFromB64("n4EPtAOCc9AlkeQHPzHStgAbgs7bTZLwUBZdR8_KuKPEHLd4rHVTeT-O-XV2jRojdNhxJWTDvNd7nqQ0VEiZQHz_AJmSCpMaJMRBSFKrKb2wqVwGU_NsYOYL-QtiWN2lbzcEe6XC0dApr5ydQLrHqkHHig3RBordaZ6Aj-oBHqFEHYpPe7Tpe-OfVfHd1E6cS6M1FZcD1NNLYD5lFHpPI9bTwJlsde3uhGqC0ZCuEHg8lhzwOHrtIQbS0FVbb9k3-tVTU4fg_3L_vniUFAKwuCLqKnS2BYwdq_mzSnbLY7h_qixoR7jig3__kRhuaxwUkRz5iaiQkqgc5gHdrNP5zw==")
    56  var e = intFromB64("AQAB")
    57  var d = bigIntFromB64("bWUC9B-EFRIo8kpGfh0ZuyGPvMNKvYWNtB_ikiH9k20eT-O1q_I78eiZkpXxXQ0UTEs2LsNRS-8uJbvQ-A1irkwMSMkK1J3XTGgdrhCku9gRldY7sNA_AKZGh-Q661_42rINLRCe8W-nZ34ui_qOfkLnK9QWDDqpaIsA-bMwWWSDFu2MUBYwkHTMEzLYGqOe04noqeq1hExBTHBOBdkMXiuFhUq1BU6l-DqEiWxqg82sXt2h-LMnT3046AOYJoRioz75tSUQfGCshWTBnP5uDjd18kKhyv07lhfSJdrPdM5Plyl21hsFf4L_mHCuoFau7gdsPfHPxxjVOcOpBrQzwQ==")
    58  var p = bigIntFromB64("uKE2dh-cTf6ERF4k4e_jy78GfPYUIaUyoSSJuBzp3Cubk3OCqs6grT8bR_cu0Dm1MZwWmtdqDyI95HrUeq3MP15vMMON8lHTeZu2lmKvwqW7anV5UzhM1iZ7z4yMkuUwFWoBvyY898EXvRD-hdqRxHlSqAZ192zB3pVFJ0s7pFc=")
    59  var q = bigIntFromB64("uKE2dh-cTf6ERF4k4e_jy78GfPYUIaUyoSSJuBzp3Cubk3OCqs6grT8bR_cu0Dm1MZwWmtdqDyI95HrUeq3MP15vMMON8lHTeZu2lmKvwqW7anV5UzhM1iZ7z4yMkuUwFWoBvyY898EXvRD-hdqRxHlSqAZ192zB3pVFJ0s7pFc=")
    60  
    61  var TheKey = rsa.PrivateKey{
    62  	PublicKey: rsa.PublicKey{N: n, E: e},
    63  	D:         d,
    64  	Primes:    []*big.Int{p, q},
    65  }
    66  
    67  var accountKey = &jose.JSONWebKey{Key: TheKey.Public()}
    68  
    69  // Return an ACME DNS identifier for the given hostname
    70  func dnsi(hostname string) identifier.ACMEIdentifier {
    71  	return identifier.DNSIdentifier(hostname)
    72  }
    73  
    74  var ctx context.Context
    75  
    76  func TestMain(m *testing.M) {
    77  	var cancel context.CancelFunc
    78  	ctx, cancel = context.WithTimeout(context.Background(), 10*time.Minute)
    79  	ret := m.Run()
    80  	cancel()
    81  	os.Exit(ret)
    82  }
    83  
    84  var accountURIPrefixes = []string{"http://boulder.service.consul:4000/acme/reg/"}
    85  
    86  func createValidationRequest(domain string, challengeType core.AcmeChallenge) *vapb.PerformValidationRequest {
    87  	return &vapb.PerformValidationRequest{
    88  		Domain: domain,
    89  		Challenge: &corepb.Challenge{
    90  			Type:              string(challengeType),
    91  			Status:            string(core.StatusPending),
    92  			Token:             expectedToken,
    93  			Validationrecords: nil,
    94  			KeyAuthorization:  expectedKeyAuthorization,
    95  		},
    96  		Authz: &vapb.AuthzMeta{
    97  			Id:    "",
    98  			RegID: 1,
    99  		},
   100  	}
   101  }
   102  
   103  func createChallenge(challengeType core.AcmeChallenge) core.Challenge {
   104  	return core.Challenge{
   105  		Type:                     challengeType,
   106  		Status:                   core.StatusPending,
   107  		Token:                    expectedToken,
   108  		ValidationRecord:         []core.ValidationRecord{},
   109  		ProvidedKeyAuthorization: expectedKeyAuthorization,
   110  	}
   111  }
   112  
   113  // setChallengeToken sets the token value, and sets the ProvidedKeyAuthorization
   114  // to match.
   115  func setChallengeToken(ch *core.Challenge, token string) {
   116  	ch.Token = token
   117  	ch.ProvidedKeyAuthorization = token + ".9jg46WB3rR_AHD-EBXdN7cBkH1WOu0tA3M9fm21mqTI"
   118  }
   119  
   120  func setup(srv *httptest.Server, maxRemoteFailures int, userAgent string, remoteVAs []RemoteVA) (*ValidationAuthorityImpl, *blog.Mock) {
   121  	features.Reset()
   122  	fc := clock.NewFake()
   123  
   124  	logger := blog.NewMock()
   125  
   126  	if userAgent == "" {
   127  		userAgent = "user agent 1.0"
   128  	}
   129  
   130  	va, err := NewValidationAuthorityImpl(
   131  		&bdns.MockClient{Log: logger},
   132  		nil,
   133  		maxRemoteFailures,
   134  		userAgent,
   135  		"letsencrypt.org",
   136  		metrics.NoopRegisterer,
   137  		fc,
   138  		logger,
   139  		accountURIPrefixes,
   140  	)
   141  
   142  	// Adjusting industry regulated ACME challenge port settings is fine during
   143  	// testing
   144  	if srv != nil {
   145  		port := getPort(srv)
   146  		va.httpPort = port
   147  		va.tlsPort = port
   148  	}
   149  
   150  	if err != nil {
   151  		panic(fmt.Sprintf("Failed to create validation authority: %v", err))
   152  	}
   153  	if remoteVAs != nil {
   154  		va.remoteVAs = remoteVAs
   155  	}
   156  	return va, logger
   157  }
   158  
   159  func setupRemote(srv *httptest.Server, userAgent string) vapb.VAClient {
   160  	innerVA, _ := setup(srv, 0, userAgent, nil)
   161  	return &localRemoteVA{remote: *innerVA}
   162  }
   163  
   164  type multiSrv struct {
   165  	*httptest.Server
   166  
   167  	mu         sync.Mutex
   168  	allowedUAs map[string]bool
   169  }
   170  
   171  func (s *multiSrv) setAllowedUAs(allowedUAs map[string]bool) {
   172  	s.mu.Lock()
   173  	defer s.mu.Unlock()
   174  	s.allowedUAs = allowedUAs
   175  }
   176  
   177  const slowRemoteSleepMillis = 1000
   178  
   179  func httpMultiSrv(t *testing.T, token string, allowedUAs map[string]bool) *multiSrv {
   180  	t.Helper()
   181  	m := http.NewServeMux()
   182  
   183  	server := httptest.NewUnstartedServer(m)
   184  	ms := &multiSrv{server, sync.Mutex{}, allowedUAs}
   185  
   186  	m.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
   187  		if r.UserAgent() == "slow remote" {
   188  			time.Sleep(slowRemoteSleepMillis)
   189  		}
   190  		ms.mu.Lock()
   191  		defer ms.mu.Unlock()
   192  		if ms.allowedUAs[r.UserAgent()] {
   193  			ch := core.Challenge{Token: token}
   194  			keyAuthz, _ := ch.ExpectedKeyAuthorization(accountKey)
   195  			fmt.Fprint(w, keyAuthz, "\n\r \t")
   196  		} else {
   197  			fmt.Fprint(w, "???")
   198  		}
   199  	})
   200  
   201  	ms.Start()
   202  	return ms
   203  }
   204  
   205  // cancelledVA is a mock that always returns context.Canceled for
   206  // PerformValidation calls
   207  type cancelledVA struct{}
   208  
   209  func (v cancelledVA) PerformValidation(_ context.Context, _ *vapb.PerformValidationRequest, _ ...grpc.CallOption) (*vapb.ValidationResult, error) {
   210  	return nil, context.Canceled
   211  }
   212  
   213  // brokenRemoteVA is a mock for the vapb.VAClient interface mocked to
   214  // always return errors.
   215  type brokenRemoteVA struct{}
   216  
   217  // errBrokenRemoteVA is the error returned by a brokenRemoteVA's
   218  // PerformValidation and IsSafeDomain functions.
   219  var errBrokenRemoteVA = errors.New("brokenRemoteVA is broken")
   220  
   221  // PerformValidation returns errBrokenRemoteVA unconditionally
   222  func (b brokenRemoteVA) PerformValidation(_ context.Context, _ *vapb.PerformValidationRequest, _ ...grpc.CallOption) (*vapb.ValidationResult, error) {
   223  	return nil, errBrokenRemoteVA
   224  }
   225  
   226  // localRemoteVA is a wrapper which fulfills the VAClient interface, but then
   227  // forwards requests directly to its inner ValidationAuthorityImpl rather than
   228  // over the network. This lets a local in-memory mock VA act like a remote VA.
   229  type localRemoteVA struct {
   230  	remote ValidationAuthorityImpl
   231  }
   232  
   233  func (lrva localRemoteVA) PerformValidation(ctx context.Context, req *vapb.PerformValidationRequest, _ ...grpc.CallOption) (*vapb.ValidationResult, error) {
   234  	return lrva.remote.PerformValidation(ctx, req)
   235  }
   236  
   237  func TestValidateMalformedChallenge(t *testing.T) {
   238  	va, _ := setup(nil, 0, "", nil)
   239  
   240  	_, prob := va.validateChallenge(ctx, dnsi("example.com"), createChallenge("fake-type-01"))
   241  
   242  	test.AssertEquals(t, prob.Type, probs.MalformedProblem)
   243  }
   244  
   245  func TestPerformValidationInvalid(t *testing.T) {
   246  	va, _ := setup(nil, 0, "", nil)
   247  
   248  	req := createValidationRequest("foo.com", core.ChallengeTypeDNS01)
   249  	res, _ := va.PerformValidation(context.Background(), req)
   250  	test.Assert(t, res.Problems != nil, "validation succeeded")
   251  
   252  	test.AssertMetricWithLabelsEquals(t, va.metrics.validationTime, prometheus.Labels{
   253  		"type":         "dns-01",
   254  		"result":       "invalid",
   255  		"problem_type": "unauthorized",
   256  	}, 1)
   257  }
   258  
   259  func TestPerformValidationValid(t *testing.T) {
   260  	va, mockLog := setup(nil, 0, "", nil)
   261  
   262  	// create a challenge with well known token
   263  	req := createValidationRequest("good-dns01.com", core.ChallengeTypeDNS01)
   264  	res, _ := va.PerformValidation(context.Background(), req)
   265  	test.Assert(t, res.Problems == nil, fmt.Sprintf("validation failed: %#v", res.Problems))
   266  
   267  	test.AssertMetricWithLabelsEquals(t, va.metrics.validationTime, prometheus.Labels{
   268  		"type":         "dns-01",
   269  		"result":       "valid",
   270  		"problem_type": "",
   271  	}, 1)
   272  	resultLog := mockLog.GetAllMatching(`Validation result`)
   273  	if len(resultLog) != 1 {
   274  		t.Fatalf("Wrong number of matching lines for 'Validation result'")
   275  	}
   276  	if !strings.Contains(resultLog[0], `"Hostname":"good-dns01.com"`) {
   277  		t.Error("PerformValidation didn't log validation hostname.")
   278  	}
   279  }
   280  
   281  // TestPerformValidationWildcard tests that the VA properly strips the `*.`
   282  // prefix from a wildcard name provided to the PerformValidation function.
   283  func TestPerformValidationWildcard(t *testing.T) {
   284  	va, mockLog := setup(nil, 0, "", nil)
   285  
   286  	// create a challenge with well known token
   287  	req := createValidationRequest("*.good-dns01.com", core.ChallengeTypeDNS01)
   288  	// perform a validation for a wildcard name
   289  	res, _ := va.PerformValidation(context.Background(), req)
   290  	test.Assert(t, res.Problems == nil, fmt.Sprintf("validation failed: %#v", res.Problems))
   291  
   292  	test.AssertMetricWithLabelsEquals(t, va.metrics.validationTime, prometheus.Labels{
   293  		"type":         "dns-01",
   294  		"result":       "valid",
   295  		"problem_type": "",
   296  	}, 1)
   297  	resultLog := mockLog.GetAllMatching(`Validation result`)
   298  	if len(resultLog) != 1 {
   299  		t.Fatalf("Wrong number of matching lines for 'Validation result'")
   300  	}
   301  
   302  	// We expect that the top level Hostname reflect the wildcard name
   303  	if !strings.Contains(resultLog[0], `"Hostname":"*.good-dns01.com"`) {
   304  		t.Errorf("PerformValidation didn't log correct validation hostname.")
   305  	}
   306  	// We expect that the ValidationRecord contain the correct non-wildcard
   307  	// hostname that was validated
   308  	if !strings.Contains(resultLog[0], `"hostname":"good-dns01.com"`) {
   309  		t.Errorf("PerformValidation didn't log correct validation record hostname.")
   310  	}
   311  }
   312  
   313  func TestDCVAndCAASequencing(t *testing.T) {
   314  	va, mockLog := setup(nil, 0, "", nil)
   315  
   316  	// When performing validation without the CAAAfterValidation flag, CAA should
   317  	// be checked.
   318  	req := createValidationRequest("good-dns01.com", core.ChallengeTypeDNS01)
   319  	res, err := va.PerformValidation(context.Background(), req)
   320  	test.AssertNotError(t, err, "performing validation")
   321  	test.Assert(t, res.Problems == nil, fmt.Sprintf("validation failed: %#v", res.Problems))
   322  	caaLog := mockLog.GetAllMatching(`Checked CAA records for`)
   323  	test.AssertEquals(t, len(caaLog), 1)
   324  
   325  	_ = features.Set(map[string]bool{features.CAAAfterValidation.String(): true})
   326  	defer features.Reset()
   327  
   328  	// When performing successful validation with the CAAAfterValidation flag,
   329  	// CAA should be checked.
   330  	mockLog.Clear()
   331  	req = createValidationRequest("good-dns01.com", core.ChallengeTypeDNS01)
   332  	res, err = va.PerformValidation(context.Background(), req)
   333  	test.AssertNotError(t, err, "performing validation")
   334  	test.Assert(t, res.Problems == nil, fmt.Sprintf("validation failed: %#v", res.Problems))
   335  	caaLog = mockLog.GetAllMatching(`Checked CAA records for`)
   336  	test.AssertEquals(t, len(caaLog), 1)
   337  
   338  	// When performing failed validation with the CAAAfterValidation flag,
   339  	// CAA should be skipped
   340  	mockLog.Clear()
   341  	req = createValidationRequest("bad-dns01.com", core.ChallengeTypeDNS01)
   342  	res, err = va.PerformValidation(context.Background(), req)
   343  	test.AssertNotError(t, err, "performing validation")
   344  	test.Assert(t, res.Problems != nil, "validation succeeded")
   345  	caaLog = mockLog.GetAllMatching(`Checked CAA records for`)
   346  	test.AssertEquals(t, len(caaLog), 0)
   347  }
   348  
   349  func TestMultiVA(t *testing.T) {
   350  	// Create a new challenge to use for the httpSrv
   351  	req := createValidationRequest("localhost", core.ChallengeTypeHTTP01)
   352  
   353  	const (
   354  		remoteUA1 = "remote 1"
   355  		remoteUA2 = "remote 2"
   356  		localUA   = "local 1"
   357  	)
   358  	allowedUAs := map[string]bool{
   359  		localUA:   true,
   360  		remoteUA1: true,
   361  		remoteUA2: true,
   362  	}
   363  
   364  	// Create an IPv4 test server
   365  	ms := httpMultiSrv(t, expectedToken, allowedUAs)
   366  	defer ms.Close()
   367  
   368  	remoteVA1 := setupRemote(ms.Server, remoteUA1)
   369  	remoteVA2 := setupRemote(ms.Server, remoteUA2)
   370  
   371  	remoteVAs := []RemoteVA{
   372  		{remoteVA1, remoteUA1},
   373  		{remoteVA2, remoteUA2},
   374  	}
   375  
   376  	enforceMultiVA := map[string]bool{
   377  		"EnforceMultiVA": true,
   378  	}
   379  	enforceMultiVAFullResults := map[string]bool{
   380  		"EnforceMultiVA":     true,
   381  		"MultiVAFullResults": true,
   382  	}
   383  	noEnforceMultiVA := map[string]bool{
   384  		"EnforceMultiVA": false,
   385  	}
   386  	noEnforceMultiVAFullResults := map[string]bool{
   387  		"EnforceMultiVA":     false,
   388  		"MultiVAFullResults": true,
   389  	}
   390  
   391  	unauthorized := probs.Unauthorized(fmt.Sprintf(
   392  		`The key authorization file from the server did not match this challenge. Expected %q (got "???")`,
   393  		expectedKeyAuthorization))
   394  
   395  	expectedInternalErrLine := fmt.Sprintf(
   396  		`ERR: \[AUDIT\] Remote VA "broken".PerformValidation failed: %s`,
   397  		errBrokenRemoteVA.Error())
   398  
   399  	testCases := []struct {
   400  		Name         string
   401  		RemoteVAs    []RemoteVA
   402  		AllowedUAs   map[string]bool
   403  		Features     map[string]bool
   404  		ExpectedProb *probs.ProblemDetails
   405  		ExpectedLog  string
   406  	}{
   407  		{
   408  			// With local and both remote VAs working there should be no problem.
   409  			Name:       "Local and remote VAs OK, enforce multi VA",
   410  			RemoteVAs:  remoteVAs,
   411  			AllowedUAs: allowedUAs,
   412  			Features:   enforceMultiVA,
   413  		},
   414  		{
   415  			// Ditto if multi VA enforcement is disabled
   416  			Name:       "Local and remote VAs OK, no enforce multi VA",
   417  			RemoteVAs:  remoteVAs,
   418  			AllowedUAs: allowedUAs,
   419  			Features:   noEnforceMultiVA,
   420  		},
   421  		{
   422  			// If the local VA fails everything should fail
   423  			Name:         "Local VA bad, remote VAs OK, no enforce multi VA",
   424  			RemoteVAs:    remoteVAs,
   425  			AllowedUAs:   map[string]bool{remoteUA1: true, remoteUA2: true},
   426  			Features:     noEnforceMultiVA,
   427  			ExpectedProb: unauthorized,
   428  		},
   429  		{
   430  			// Ditto when enforcing remote VA
   431  			Name:         "Local VA bad, remote VAs OK, enforce multi VA",
   432  			RemoteVAs:    remoteVAs,
   433  			AllowedUAs:   map[string]bool{remoteUA1: true, remoteUA2: true},
   434  			Features:     enforceMultiVA,
   435  			ExpectedProb: unauthorized,
   436  		},
   437  		{
   438  			// If a remote VA fails with an internal err it should fail when enforcing multi VA
   439  			Name: "Local VA ok, remote VA internal err, enforce multi VA",
   440  			RemoteVAs: []RemoteVA{
   441  				{remoteVA1, remoteUA1},
   442  				{&brokenRemoteVA{}, "broken"},
   443  			},
   444  			AllowedUAs:   allowedUAs,
   445  			Features:     enforceMultiVA,
   446  			ExpectedProb: probs.ServerInternal("During secondary validation: Remote PerformValidation RPC failed"),
   447  			// The real failure cause should be logged
   448  			ExpectedLog: expectedInternalErrLine,
   449  		},
   450  		{
   451  			// If a remote VA fails with an internal err it should not fail when not
   452  			// enforcing multi VA
   453  			Name: "Local VA ok, remote VA internal err, no enforce multi VA",
   454  			RemoteVAs: []RemoteVA{
   455  				{remoteVA1, remoteUA1},
   456  				{&brokenRemoteVA{}, "broken"},
   457  			},
   458  			AllowedUAs: allowedUAs,
   459  			Features:   noEnforceMultiVA,
   460  			// Like above, the real failure cause will be logged eventually, but that
   461  			// will happen asynchronously. It's not guaranteed to happen before the
   462  			// test case exits, so we don't check for it here.
   463  		},
   464  		{
   465  			// With only one working remote VA there should *not* be a validation
   466  			// failure when not enforcing multi VA.
   467  			Name:       "Local VA and one remote VA OK, no enforce multi VA",
   468  			RemoteVAs:  remoteVAs,
   469  			AllowedUAs: map[string]bool{localUA: true, remoteUA2: true},
   470  			Features:   noEnforceMultiVA,
   471  		},
   472  		{
   473  			// With only one working remote VA there should be a validation failure
   474  			// when enforcing multi VA.
   475  			Name:       "Local VA and one remote VA OK, enforce multi VA",
   476  			RemoteVAs:  remoteVAs,
   477  			AllowedUAs: map[string]bool{localUA: true, remoteUA2: true},
   478  			Features:   enforceMultiVA,
   479  			ExpectedProb: probs.Unauthorized(fmt.Sprintf(
   480  				`During secondary validation: The key authorization file from the server did not match this challenge. Expected %q (got "???")`,
   481  				expectedKeyAuthorization)),
   482  		},
   483  		{
   484  			// When enforcing multi-VA, any cancellations are a problem.
   485  			Name: "Local VA and one remote VA OK, one cancelled VA, enforce multi VA",
   486  			RemoteVAs: []RemoteVA{
   487  				{remoteVA1, remoteUA1},
   488  				{cancelledVA{}, remoteUA2},
   489  			},
   490  			AllowedUAs:   allowedUAs,
   491  			Features:     enforceMultiVA,
   492  			ExpectedProb: probs.ServerInternal("During secondary validation: Remote PerformValidation RPC canceled"),
   493  		},
   494  		{
   495  			// When enforcing multi-VA, any cancellations are a problem.
   496  			Name: "Local VA OK, two cancelled remote VAs, enforce multi VA",
   497  			RemoteVAs: []RemoteVA{
   498  				{cancelledVA{}, remoteUA1},
   499  				{cancelledVA{}, remoteUA2},
   500  			},
   501  			AllowedUAs:   allowedUAs,
   502  			Features:     enforceMultiVA,
   503  			ExpectedProb: probs.ServerInternal("During secondary validation: Remote PerformValidation RPC canceled"),
   504  		},
   505  		{
   506  			// With the local and remote VAs seeing diff problems and the full results
   507  			// feature flag on but multi VA enforcement off we expect
   508  			// no problem.
   509  			Name:       "Local and remote VA differential, full results, no enforce multi VA",
   510  			RemoteVAs:  remoteVAs,
   511  			AllowedUAs: map[string]bool{localUA: true},
   512  			Features:   noEnforceMultiVAFullResults,
   513  		},
   514  		{
   515  			// With the local and remote VAs seeing diff problems and the full results
   516  			// feature flag on and multi VA enforcement on we expect a problem.
   517  			Name:       "Local and remote VA differential, full results, enforce multi VA",
   518  			RemoteVAs:  remoteVAs,
   519  			AllowedUAs: map[string]bool{localUA: true},
   520  			Features:   enforceMultiVAFullResults,
   521  			ExpectedProb: probs.Unauthorized(fmt.Sprintf(
   522  				`During secondary validation: The key authorization file from the server did not match this challenge. Expected %q (got "???")`,
   523  				expectedKeyAuthorization)),
   524  		},
   525  	}
   526  
   527  	for _, tc := range testCases {
   528  		t.Run(tc.Name, func(t *testing.T) {
   529  			// Configure the test server with the testcase allowed UAs.
   530  			ms.setAllowedUAs(tc.AllowedUAs)
   531  
   532  			// Configure a primary VA with testcase remote VAs.
   533  			localVA, mockLog := setup(ms.Server, 0, localUA, tc.RemoteVAs)
   534  
   535  			if tc.Features != nil {
   536  				err := features.Set(tc.Features)
   537  				test.AssertNotError(t, err, "Failed to set feature flags")
   538  				defer features.Reset()
   539  			}
   540  
   541  			// Perform all validations
   542  			res, _ := localVA.PerformValidation(ctx, req)
   543  			if res.Problems == nil && tc.ExpectedProb != nil {
   544  				t.Errorf("expected prob %v, got nil", tc.ExpectedProb)
   545  			} else if res.Problems != nil && tc.ExpectedProb == nil {
   546  				t.Errorf("expected no prob, got %v", res.Problems)
   547  			} else if res.Problems != nil && tc.ExpectedProb != nil {
   548  				// That result should match expected.
   549  				test.AssertEquals(t, res.Problems.ProblemType, string(tc.ExpectedProb.Type))
   550  				test.AssertEquals(t, res.Problems.Detail, tc.ExpectedProb.Detail)
   551  			}
   552  
   553  			if tc.ExpectedLog != "" {
   554  				lines := mockLog.GetAllMatching(tc.ExpectedLog)
   555  				if len(lines) != 1 {
   556  					t.Fatalf("Got log %v; expected %q", mockLog.GetAll(), tc.ExpectedLog)
   557  				}
   558  			}
   559  		})
   560  	}
   561  }
   562  
   563  func TestMultiVAEarlyReturn(t *testing.T) {
   564  	const (
   565  		remoteUA1 = "remote 1"
   566  		remoteUA2 = "slow remote"
   567  		localUA   = "local 1"
   568  	)
   569  	allowedUAs := map[string]bool{
   570  		localUA:   true,
   571  		remoteUA1: false, // forbid UA 1 to provoke early return
   572  		remoteUA2: true,
   573  	}
   574  
   575  	ms := httpMultiSrv(t, expectedToken, allowedUAs)
   576  	defer ms.Close()
   577  
   578  	remoteVA1 := setupRemote(ms.Server, remoteUA1)
   579  	remoteVA2 := setupRemote(ms.Server, remoteUA2)
   580  
   581  	remoteVAs := []RemoteVA{
   582  		{remoteVA1, remoteUA1},
   583  		{remoteVA2, remoteUA2},
   584  	}
   585  
   586  	// Create a local test VA with the two remote VAs
   587  	localVA, mockLog := setup(ms.Server, 0, localUA, remoteVAs)
   588  
   589  	testCases := []struct {
   590  		Name        string
   591  		EarlyReturn bool
   592  	}{
   593  		{
   594  			Name: "One slow remote VA, no early return",
   595  		},
   596  		{
   597  			Name:        "One slow remote VA, early return",
   598  			EarlyReturn: true,
   599  		},
   600  	}
   601  
   602  	earlyReturnFeatures := map[string]bool{
   603  		"EnforceMultiVA":     true,
   604  		"MultiVAFullResults": false,
   605  	}
   606  	noEarlyReturnFeatures := map[string]bool{
   607  		"EnforceMultiVA":     true,
   608  		"MultiVAFullResults": true,
   609  	}
   610  
   611  	req := createValidationRequest("localhost", core.ChallengeTypeHTTP01)
   612  	for _, tc := range testCases {
   613  		t.Run(tc.Name, func(t *testing.T) {
   614  			mockLog.Clear()
   615  
   616  			var err error
   617  			if tc.EarlyReturn {
   618  				err = features.Set(earlyReturnFeatures)
   619  			} else {
   620  				err = features.Set(noEarlyReturnFeatures)
   621  			}
   622  			test.AssertNotError(t, err, "Failed to set MultiVAFullResults feature flag")
   623  			defer features.Reset()
   624  
   625  			start := time.Now()
   626  
   627  			// Perform all validations
   628  			res, _ := localVA.PerformValidation(ctx, req)
   629  			// It should always fail
   630  			if res.Problems == nil {
   631  				t.Error("expected prob from PerformValidation, got nil")
   632  			}
   633  
   634  			elapsed := time.Since(start).Round(time.Millisecond).Milliseconds()
   635  
   636  			// The slow UA should sleep for `slowRemoteSleepMillis`. In the early return
   637  			// case the first remote VA should fail the overall validation and a prob
   638  			// should be returned quickly (i.e. in less than half of `slowRemoteSleepMillis`).
   639  			// In the non-early return case we don't expect a problem until
   640  			// `slowRemoteSleepMillis`.
   641  			if tc.EarlyReturn && elapsed > slowRemoteSleepMillis/2 {
   642  				t.Errorf(
   643  					"Expected an early return from PerformValidation in < %d ms, took %d ms",
   644  					slowRemoteSleepMillis/2, elapsed)
   645  			}
   646  		})
   647  	}
   648  }
   649  
   650  func TestMultiVAPolicy(t *testing.T) {
   651  	const (
   652  		remoteUA1 = "remote 1"
   653  		remoteUA2 = "remote 2"
   654  		localUA   = "local 1"
   655  	)
   656  	// Forbid both remote UAs to ensure that multi-va fails
   657  	allowedUAs := map[string]bool{
   658  		localUA:   true,
   659  		remoteUA1: false,
   660  		remoteUA2: false,
   661  	}
   662  
   663  	ms := httpMultiSrv(t, expectedToken, allowedUAs)
   664  	defer ms.Close()
   665  
   666  	remoteVA1 := setupRemote(ms.Server, remoteUA1)
   667  	remoteVA2 := setupRemote(ms.Server, remoteUA2)
   668  
   669  	remoteVAs := []RemoteVA{
   670  		{remoteVA1, remoteUA1},
   671  		{remoteVA2, remoteUA2},
   672  	}
   673  
   674  	// Create a local test VA with the two remote VAs
   675  	localVA, _ := setup(ms.Server, 0, localUA, remoteVAs)
   676  
   677  	// Ensure multi VA enforcement is enabled, don't wait for full multi VA
   678  	// results.
   679  	err := features.Set(map[string]bool{
   680  		"EnforceMultiVA":     true,
   681  		"MultiVAFullResults": false,
   682  	})
   683  	test.AssertNotError(t, err, "setting feature flags")
   684  	defer features.Reset()
   685  
   686  	// Perform validation for a domain not in the disabledDomains list
   687  	req := createValidationRequest("letsencrypt.org", core.ChallengeTypeHTTP01)
   688  	res, _ := localVA.PerformValidation(ctx, req)
   689  	// It should fail
   690  	if res.Problems == nil {
   691  		t.Error("expected prob from PerformValidation, got nil")
   692  	}
   693  }
   694  
   695  func TestDetailedError(t *testing.T) {
   696  	cases := []struct {
   697  		err      error
   698  		ip       net.IP
   699  		expected string
   700  	}{
   701  		{
   702  			err: ipError{
   703  				ip: net.ParseIP("192.168.1.1"),
   704  				err: &net.OpError{
   705  					Op:  "dial",
   706  					Net: "tcp",
   707  					Err: &os.SyscallError{
   708  						Syscall: "getsockopt",
   709  						Err:     syscall.ECONNREFUSED,
   710  					},
   711  				},
   712  			},
   713  			expected: "192.168.1.1: Connection refused",
   714  		},
   715  		{
   716  			err: &net.OpError{
   717  				Op:  "dial",
   718  				Net: "tcp",
   719  				Err: &os.SyscallError{
   720  					Syscall: "getsockopt",
   721  					Err:     syscall.ECONNREFUSED,
   722  				},
   723  			},
   724  			expected: "Connection refused",
   725  		},
   726  		{
   727  			err: &net.OpError{
   728  				Op:  "dial",
   729  				Net: "tcp",
   730  				Err: &os.SyscallError{
   731  					Syscall: "getsockopt",
   732  					Err:     syscall.ECONNRESET,
   733  				},
   734  			},
   735  			ip:       nil,
   736  			expected: "Connection reset by peer",
   737  		},
   738  	}
   739  	for _, tc := range cases {
   740  		actual := detailedError(tc.err).Detail
   741  		if actual != tc.expected {
   742  			t.Errorf("Wrong detail for %v. Got %q, expected %q", tc.err, actual, tc.expected)
   743  		}
   744  	}
   745  }
   746  
   747  func TestLogRemoteValidationDifferentials(t *testing.T) {
   748  	// Create some remote VAs
   749  	remoteVA1 := setupRemote(nil, "remote 1")
   750  	remoteVA2 := setupRemote(nil, "remote 2")
   751  	remoteVA3 := setupRemote(nil, "remote 3")
   752  	remoteVAs := []RemoteVA{
   753  		{remoteVA1, "remote 1"},
   754  		{remoteVA2, "remote 2"},
   755  		{remoteVA3, "remote 3"},
   756  	}
   757  
   758  	// Set up a local VA that allows a max of 2 remote failures.
   759  	localVA, mockLog := setup(nil, 2, "local 1", remoteVAs)
   760  
   761  	egProbA := probs.DNS("root DNS servers closed at 4:30pm")
   762  	egProbB := probs.OrderNotReady("please take a number")
   763  
   764  	testCases := []struct {
   765  		name          string
   766  		primaryResult *probs.ProblemDetails
   767  		remoteProbs   []*remoteValidationResult
   768  		expectedLog   string
   769  	}{
   770  		{
   771  			name:          "remote and primary results equal (all nil)",
   772  			primaryResult: nil,
   773  			remoteProbs: []*remoteValidationResult{
   774  				{Problem: nil, VAHostname: "remoteA"},
   775  				{Problem: nil, VAHostname: "remoteB"},
   776  				{Problem: nil, VAHostname: "remoteC"},
   777  			},
   778  		},
   779  		{
   780  			name:          "remote and primary results equal (not nil)",
   781  			primaryResult: egProbA,
   782  			remoteProbs: []*remoteValidationResult{
   783  				{Problem: egProbA, VAHostname: "remoteA"},
   784  				{Problem: egProbA, VAHostname: "remoteB"},
   785  				{Problem: egProbA, VAHostname: "remoteC"},
   786  			},
   787  		},
   788  		{
   789  			name:          "remote and primary differ (primary nil)",
   790  			primaryResult: nil,
   791  			remoteProbs: []*remoteValidationResult{
   792  				{Problem: egProbA, VAHostname: "remoteA"},
   793  				{Problem: nil, VAHostname: "remoteB"},
   794  				{Problem: egProbB, VAHostname: "remoteC"},
   795  			},
   796  			expectedLog: `INFO: remoteVADifferentials JSON={"Domain":"example.com","AccountID":1999,"ChallengeType":"blorpus-01","PrimaryResult":null,"RemoteSuccesses":1,"RemoteFailures":[{"VAHostname":"remoteA","Problem":{"type":"dns","detail":"root DNS servers closed at 4:30pm","status":400}},{"VAHostname":"remoteC","Problem":{"type":"orderNotReady","detail":"please take a number","status":403}}]}`,
   797  		},
   798  		{
   799  			name:          "remote and primary differ (primary not nil)",
   800  			primaryResult: egProbA,
   801  			remoteProbs: []*remoteValidationResult{
   802  				{Problem: nil, VAHostname: "remoteA"},
   803  				{Problem: egProbB, VAHostname: "remoteB"},
   804  				{Problem: nil, VAHostname: "remoteC"},
   805  			},
   806  			expectedLog: `INFO: remoteVADifferentials JSON={"Domain":"example.com","AccountID":1999,"ChallengeType":"blorpus-01","PrimaryResult":{"type":"dns","detail":"root DNS servers closed at 4:30pm","status":400},"RemoteSuccesses":2,"RemoteFailures":[{"VAHostname":"remoteB","Problem":{"type":"orderNotReady","detail":"please take a number","status":403}}]}`,
   807  		},
   808  	}
   809  
   810  	for _, tc := range testCases {
   811  		t.Run(tc.name, func(t *testing.T) {
   812  			mockLog.Clear()
   813  
   814  			localVA.logRemoteValidationDifferentials(
   815  				"example.com", 1999, "blorpus-01", tc.primaryResult, tc.remoteProbs)
   816  
   817  			lines := mockLog.GetAllMatching("remoteVADifferentials JSON=.*")
   818  			if tc.expectedLog != "" {
   819  				test.AssertEquals(t, len(lines), 1)
   820  				test.AssertEquals(t, lines[0], tc.expectedLog)
   821  			} else {
   822  				test.AssertEquals(t, len(lines), 0)
   823  			}
   824  		})
   825  	}
   826  }
   827  

View as plain text