1 package notmain
2
3 import (
4 "bytes"
5 "context"
6 "crypto/x509"
7 "flag"
8 "fmt"
9 "html/template"
10 netmail "net/mail"
11 "os"
12 "strings"
13 "time"
14
15 "github.com/jmhodges/clock"
16 "github.com/prometheus/client_golang/prometheus"
17 "golang.org/x/crypto/ocsp"
18 "google.golang.org/grpc"
19 "google.golang.org/protobuf/types/known/emptypb"
20
21 "github.com/letsencrypt/boulder/cmd"
22 "github.com/letsencrypt/boulder/config"
23 "github.com/letsencrypt/boulder/core"
24 "github.com/letsencrypt/boulder/db"
25 bgrpc "github.com/letsencrypt/boulder/grpc"
26 blog "github.com/letsencrypt/boulder/log"
27 "github.com/letsencrypt/boulder/mail"
28 rapb "github.com/letsencrypt/boulder/ra/proto"
29 "github.com/letsencrypt/boulder/sa"
30 )
31
32 const blockedKeysGaugeLimit = 1000
33
34 var keysToProcess = prometheus.NewGauge(prometheus.GaugeOpts{
35 Name: "bad_keys_to_process",
36 Help: fmt.Sprintf("A gauge of blockedKeys rows to process (max: %d)", blockedKeysGaugeLimit),
37 })
38 var keysProcessed = prometheus.NewCounterVec(prometheus.CounterOpts{
39 Name: "bad_keys_processed",
40 Help: "A counter of blockedKeys rows processed labelled by processing state",
41 }, []string{"state"})
42 var certsRevoked = prometheus.NewCounter(prometheus.CounterOpts{
43 Name: "bad_keys_certs_revoked",
44 Help: "A counter of certificates associated with rows in blockedKeys that have been revoked",
45 })
46 var mailErrors = prometheus.NewCounter(prometheus.CounterOpts{
47 Name: "bad_keys_mail_errors",
48 Help: "A counter of email send errors",
49 })
50
51
52
53
54 type revoker interface {
55 AdministrativelyRevokeCertificate(ctx context.Context, in *rapb.AdministrativelyRevokeCertificateRequest, opts ...grpc.CallOption) (*emptypb.Empty, error)
56 }
57
58 type badKeyRevoker struct {
59 dbMap *db.WrappedMap
60 maxRevocations int
61 serialBatchSize int
62 raClient revoker
63 mailer mail.Mailer
64 emailSubject string
65 emailTemplate *template.Template
66 logger blog.Logger
67 clk clock.Clock
68 backoffIntervalBase time.Duration
69 backoffIntervalMax time.Duration
70 backoffFactor float64
71 backoffTicker int
72 }
73
74
75 type uncheckedBlockedKey struct {
76 KeyHash []byte
77 RevokedBy int64
78 }
79
80 func (ubk uncheckedBlockedKey) String() string {
81 return fmt.Sprintf("[revokedBy: %d, keyHash: %x]",
82 ubk.RevokedBy, ubk.KeyHash)
83 }
84
85 func (bkr *badKeyRevoker) countUncheckedKeys(ctx context.Context) (int, error) {
86 var count int
87 err := bkr.dbMap.SelectOne(
88 ctx,
89 &count,
90 `SELECT COUNT(*)
91 FROM (SELECT 1 FROM blockedKeys
92 WHERE extantCertificatesChecked = false
93 LIMIT ?) AS a`,
94 blockedKeysGaugeLimit,
95 )
96 return count, err
97 }
98
99 func (bkr *badKeyRevoker) selectUncheckedKey(ctx context.Context) (uncheckedBlockedKey, error) {
100 var row uncheckedBlockedKey
101 err := bkr.dbMap.SelectOne(
102 ctx,
103 &row,
104 `SELECT keyHash, revokedBy
105 FROM blockedKeys
106 WHERE extantCertificatesChecked = false
107 LIMIT 1`,
108 )
109 return row, err
110 }
111
112
113 type unrevokedCertificate struct {
114 ID int
115 Serial string
116 DER []byte
117 RegistrationID int64
118 Status core.OCSPStatus
119 IsExpired bool
120 }
121
122 func (uc unrevokedCertificate) String() string {
123 return fmt.Sprintf("id=%d serial=%s regID=%d status=%s expired=%t",
124 uc.ID, uc.Serial, uc.RegistrationID, uc.Status, uc.IsExpired)
125 }
126
127
128
129
130 func (bkr *badKeyRevoker) findUnrevoked(ctx context.Context, unchecked uncheckedBlockedKey) ([]unrevokedCertificate, error) {
131 var unrevokedCerts []unrevokedCertificate
132 initialID := 0
133 for {
134 var batch []struct {
135 ID int
136 CertSerial string
137 }
138 _, err := bkr.dbMap.Select(
139 ctx,
140 &batch,
141 "SELECT id, certSerial FROM keyHashToSerial WHERE keyHash = ? AND id > ? AND certNotAfter > ? ORDER BY id LIMIT ?",
142 unchecked.KeyHash,
143 initialID,
144 bkr.clk.Now(),
145 bkr.serialBatchSize,
146 )
147 if err != nil {
148 return nil, err
149 }
150 if len(batch) == 0 {
151 break
152 }
153 initialID = batch[len(batch)-1].ID
154 for _, serial := range batch {
155 var unrevokedCert unrevokedCertificate
156
157
158
159
160 err = bkr.dbMap.SelectOne(
161 ctx,
162 &unrevokedCert,
163 `SELECT cs.id, cs.serial, c.registrationID, c.der, cs.status, cs.isExpired
164 FROM certificateStatus AS cs
165 JOIN precertificates AS c
166 ON cs.serial = c.serial
167 WHERE cs.serial = ?
168 LIMIT 1`,
169 serial.CertSerial,
170 )
171 if err != nil {
172 return nil, err
173 }
174 if unrevokedCert.IsExpired || unrevokedCert.Status == core.OCSPStatusRevoked {
175 continue
176 }
177 unrevokedCerts = append(unrevokedCerts, unrevokedCert)
178 }
179 }
180 if len(unrevokedCerts) > bkr.maxRevocations {
181 return nil, fmt.Errorf("too many certificates to revoke associated with %x: got %d, max %d", unchecked.KeyHash, len(unrevokedCerts), bkr.maxRevocations)
182 }
183 return unrevokedCerts, nil
184 }
185
186
187
188 func (bkr *badKeyRevoker) markRowChecked(ctx context.Context, unchecked uncheckedBlockedKey) error {
189 _, err := bkr.dbMap.ExecContext(ctx, "UPDATE blockedKeys SET extantCertificatesChecked = true WHERE keyHash = ?", unchecked.KeyHash)
190 return err
191 }
192
193
194 func (bkr *badKeyRevoker) resolveContacts(ctx context.Context, ids []int64) (map[int64][]string, error) {
195 idToEmail := map[int64][]string{}
196 for _, id := range ids {
197 var emails struct {
198 Contact []string
199 }
200 err := bkr.dbMap.SelectOne(ctx, &emails, "SELECT contact FROM registrations WHERE id = ?", id)
201 if err != nil {
202
203
204 return nil, err
205 }
206 if len(emails.Contact) != 0 {
207 for _, email := range emails.Contact {
208 idToEmail[id] = append(idToEmail[id], strings.TrimPrefix(email, "mailto:"))
209 }
210 } else {
211
212
213 idToEmail[id] = append(idToEmail[id], "")
214 continue
215 }
216 }
217 return idToEmail, nil
218 }
219
220 var maxSerials = 100
221
222
223
224 func (bkr *badKeyRevoker) sendMessage(addr string, serials []string) error {
225 conn, err := bkr.mailer.Connect()
226 if err != nil {
227 return err
228 }
229 defer func() {
230 _ = conn.Close()
231 }()
232 mutSerials := make([]string, len(serials))
233 copy(mutSerials, serials)
234 if len(mutSerials) > maxSerials {
235 more := len(mutSerials) - maxSerials
236 mutSerials = mutSerials[:maxSerials]
237 mutSerials = append(mutSerials, fmt.Sprintf("and %d more certificates.", more))
238 }
239 message := bytes.NewBuffer(nil)
240 err = bkr.emailTemplate.Execute(message, mutSerials)
241 if err != nil {
242 return err
243 }
244 err = conn.SendMail([]string{addr}, bkr.emailSubject, message.String())
245 if err != nil {
246 return err
247 }
248 return nil
249 }
250
251
252
253
254 func (bkr *badKeyRevoker) revokeCerts(revokerEmails []string, emailToCerts map[string][]unrevokedCertificate) error {
255 revokerEmailsMap := map[string]bool{}
256 for _, email := range revokerEmails {
257 revokerEmailsMap[email] = true
258 }
259
260 alreadyRevoked := map[int]bool{}
261 for email, certs := range emailToCerts {
262 var revokedSerials []string
263 for _, cert := range certs {
264 revokedSerials = append(revokedSerials, cert.Serial)
265 if alreadyRevoked[cert.ID] {
266 continue
267 }
268 _, err := bkr.raClient.AdministrativelyRevokeCertificate(context.Background(), &rapb.AdministrativelyRevokeCertificateRequest{
269 Cert: cert.DER,
270 Serial: cert.Serial,
271 Code: int64(ocsp.KeyCompromise),
272 AdminName: "bad-key-revoker",
273 })
274 if err != nil {
275 return err
276 }
277 certsRevoked.Inc()
278 alreadyRevoked[cert.ID] = true
279 }
280
281 if revokerEmailsMap[email] || email == "" {
282 continue
283 }
284 err := bkr.sendMessage(email, revokedSerials)
285 if err != nil {
286 mailErrors.Inc()
287 bkr.logger.Errf("failed to send message to %q: %s", email, err)
288 continue
289 }
290 }
291 return nil
292 }
293
294
295
296 func (bkr *badKeyRevoker) invoke(ctx context.Context) (bool, error) {
297
298 uncheckedCount, err := bkr.countUncheckedKeys(ctx)
299 if err != nil {
300 return false, err
301 }
302
303
304
305 keysToProcess.Set(float64(uncheckedCount))
306
307 if uncheckedCount >= blockedKeysGaugeLimit {
308 bkr.logger.AuditInfof("found >= %d unchecked blocked keys left to process", uncheckedCount)
309 } else {
310 bkr.logger.AuditInfof("found %d unchecked blocked keys left to process", uncheckedCount)
311 }
312
313
314 unchecked, err := bkr.selectUncheckedKey(ctx)
315 if err != nil {
316 if db.IsNoRows(err) {
317 return true, nil
318 }
319 return false, err
320 }
321 bkr.logger.AuditInfo(fmt.Sprintf("found unchecked block key to work on: %s", unchecked))
322
323
324 unrevokedCerts, err := bkr.findUnrevoked(ctx, unchecked)
325 if err != nil {
326 bkr.logger.AuditInfo(fmt.Sprintf("finding unrevoked certificates related to %s: %s",
327 unchecked, err))
328 return false, err
329 }
330 if len(unrevokedCerts) == 0 {
331 bkr.logger.AuditInfo(fmt.Sprintf("found no certificates that need revoking related to %s, marking row as checked", unchecked))
332
333 err = bkr.markRowChecked(ctx, unchecked)
334 if err != nil {
335 return false, err
336 }
337 return false, nil
338 }
339
340
341
342 ownedBy := map[int64][]unrevokedCertificate{}
343 var ids []int64
344 for _, cert := range unrevokedCerts {
345 if ownedBy[cert.RegistrationID] == nil {
346 ids = append(ids, cert.RegistrationID)
347 }
348 ownedBy[cert.RegistrationID] = append(ownedBy[cert.RegistrationID], cert)
349 }
350
351
352
353
354
355 if _, present := ownedBy[unchecked.RevokedBy]; !present && unchecked.RevokedBy != 0 {
356 ids = append(ids, unchecked.RevokedBy)
357 }
358
359 idToEmails, err := bkr.resolveContacts(ctx, ids)
360 if err != nil {
361 return false, err
362 }
363
364
365
366 emailsToCerts := map[string][]unrevokedCertificate{}
367 for id, emails := range idToEmails {
368 for _, email := range emails {
369 emailsToCerts[email] = append(emailsToCerts[email], ownedBy[id]...)
370 }
371 }
372
373 revokerEmails := idToEmails[unchecked.RevokedBy]
374 bkr.logger.AuditInfo(fmt.Sprintf("revoking certs. revoked emails=%v, emailsToCerts=%s",
375 revokerEmails, emailsToCerts))
376
377
378 err = bkr.revokeCerts(idToEmails[unchecked.RevokedBy], emailsToCerts)
379 if err != nil {
380 return false, err
381 }
382
383
384 err = bkr.markRowChecked(ctx, unchecked)
385 if err != nil {
386 return false, err
387 }
388 return false, nil
389 }
390
391 type Config struct {
392 BadKeyRevoker struct {
393 DB cmd.DBConfig
394 DebugAddr string `validate:"hostname_port"`
395
396 TLS cmd.TLSConfig
397 RAService *cmd.GRPCClientConfig
398
399
400
401
402
403 MaximumRevocations int `validate:"gte=0"`
404
405
406 FindCertificatesBatchSize int `validate:"required"`
407
408
409
410
411 Interval config.Duration `validate:"-"`
412
413
414
415
416 BackoffIntervalMax config.Duration `validate:"-"`
417
418 Mailer struct {
419 cmd.SMTPConfig
420
421
422 SMTPTrustedRootFile string
423
424 From string `validate:"required"`
425 EmailSubject string `validate:"required"`
426 EmailTemplate string `validate:"required"`
427 }
428 }
429
430 Syslog cmd.SyslogConfig
431 OpenTelemetry cmd.OpenTelemetryConfig
432 }
433
434 func main() {
435 configPath := flag.String("config", "", "File path to the configuration file for this service")
436 flag.Parse()
437
438 if *configPath == "" {
439 flag.Usage()
440 os.Exit(1)
441 }
442 var config Config
443 err := cmd.ReadConfigFile(*configPath, &config)
444 cmd.FailOnError(err, "Failed reading config file")
445
446 scope, logger, oTelShutdown := cmd.StatsAndLogging(config.Syslog, config.OpenTelemetry, config.BadKeyRevoker.DebugAddr)
447 defer oTelShutdown(context.Background())
448 logger.Info(cmd.VersionString())
449 clk := cmd.Clock()
450
451 scope.MustRegister(keysProcessed)
452 scope.MustRegister(certsRevoked)
453 scope.MustRegister(mailErrors)
454
455 dbMap, err := sa.InitWrappedDb(config.BadKeyRevoker.DB, scope, logger)
456 cmd.FailOnError(err, "While initializing dbMap")
457
458 tlsConfig, err := config.BadKeyRevoker.TLS.Load(scope)
459 cmd.FailOnError(err, "TLS config")
460
461 conn, err := bgrpc.ClientSetup(config.BadKeyRevoker.RAService, tlsConfig, scope, clk)
462 cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to RA")
463 rac := rapb.NewRegistrationAuthorityClient(conn)
464
465 var smtpRoots *x509.CertPool
466 if config.BadKeyRevoker.Mailer.SMTPTrustedRootFile != "" {
467 pem, err := os.ReadFile(config.BadKeyRevoker.Mailer.SMTPTrustedRootFile)
468 cmd.FailOnError(err, "Loading trusted roots file")
469 smtpRoots = x509.NewCertPool()
470 if !smtpRoots.AppendCertsFromPEM(pem) {
471 cmd.FailOnError(nil, "Failed to parse root certs PEM")
472 }
473 }
474
475 fromAddress, err := netmail.ParseAddress(config.BadKeyRevoker.Mailer.From)
476 cmd.FailOnError(err, fmt.Sprintf("Could not parse from address: %s", config.BadKeyRevoker.Mailer.From))
477
478 smtpPassword, err := config.BadKeyRevoker.Mailer.PasswordConfig.Pass()
479 cmd.FailOnError(err, "Failed to load SMTP password")
480 mailClient := mail.New(
481 config.BadKeyRevoker.Mailer.Server,
482 config.BadKeyRevoker.Mailer.Port,
483 config.BadKeyRevoker.Mailer.Username,
484 smtpPassword,
485 smtpRoots,
486 *fromAddress,
487 logger,
488 scope,
489 1*time.Second,
490 5*60*time.Second,
491 )
492
493 if config.BadKeyRevoker.Mailer.EmailSubject == "" {
494 cmd.Fail("BadKeyRevoker.Mailer.EmailSubject must be populated")
495 }
496 templateBytes, err := os.ReadFile(config.BadKeyRevoker.Mailer.EmailTemplate)
497 cmd.FailOnError(err, fmt.Sprintf("failed to read email template %q: %s", config.BadKeyRevoker.Mailer.EmailTemplate, err))
498 emailTemplate, err := template.New("email").Parse(string(templateBytes))
499 cmd.FailOnError(err, fmt.Sprintf("failed to parse email template %q: %s", config.BadKeyRevoker.Mailer.EmailTemplate, err))
500
501 bkr := &badKeyRevoker{
502 dbMap: dbMap,
503 maxRevocations: config.BadKeyRevoker.MaximumRevocations,
504 serialBatchSize: config.BadKeyRevoker.FindCertificatesBatchSize,
505 raClient: rac,
506 mailer: mailClient,
507 emailSubject: config.BadKeyRevoker.Mailer.EmailSubject,
508 emailTemplate: emailTemplate,
509 logger: logger,
510 clk: clk,
511 backoffIntervalMax: config.BadKeyRevoker.BackoffIntervalMax.Duration,
512 backoffIntervalBase: config.BadKeyRevoker.Interval.Duration,
513 backoffFactor: 1.3,
514 }
515
516
517
518
519 if bkr.backoffIntervalMax == 0 {
520 bkr.backoffIntervalMax = time.Second * 60
521 }
522
523
524
525 if bkr.backoffIntervalBase == 0 {
526 bkr.backoffIntervalBase = time.Second
527 }
528
529
530 for {
531 noWork, err := bkr.invoke(context.Background())
532 if err != nil {
533 keysProcessed.WithLabelValues("error").Inc()
534 logger.AuditErrf("failed to process blockedKeys row: %s", err)
535
536 bkr.backoff()
537 continue
538 }
539 if noWork {
540 logger.Info("no work to do")
541
542 bkr.backoff()
543 } else {
544 keysProcessed.WithLabelValues("success").Inc()
545
546 bkr.backoffReset()
547 }
548 }
549 }
550
551
552
553
554 func (bkr *badKeyRevoker) backoff() {
555 bkr.backoffTicker++
556 backoffDur := core.RetryBackoff(
557 bkr.backoffTicker,
558 bkr.backoffIntervalBase,
559 bkr.backoffIntervalMax,
560 bkr.backoffFactor,
561 )
562 bkr.logger.Infof("backoff trying again in %.2f seconds", backoffDur.Seconds())
563 bkr.clk.Sleep(backoffDur)
564 }
565
566
567 func (bkr *badKeyRevoker) backoffReset() {
568 bkr.backoffTicker = 0
569 }
570
571 func init() {
572 cmd.RegisterCommand("bad-key-revoker", main, &cmd.ConfigValidator{Config: &Config{}})
573 }
574
View as plain text