main.go

Documentation: github.com/letsencrypt/boulder/cmd/id-exporter

     1  package notmain
     2  
     3  import (
     4  	"bufio"
     5  	"context"
     6  	"encoding/json"
     7  	"errors"
     8  	"flag"
     9  	"fmt"
    10  	"os"
    11  	"strings"
    12  	"time"
    13  
    14  	"github.com/jmhodges/clock"
    15  	"github.com/letsencrypt/boulder/cmd"
    16  	"github.com/letsencrypt/boulder/db"
    17  	"github.com/letsencrypt/boulder/features"
    18  	blog "github.com/letsencrypt/boulder/log"
    19  	"github.com/letsencrypt/boulder/sa"
    20  )
    21  
    22  type idExporter struct {
    23  	log   blog.Logger
    24  	dbMap *db.WrappedMap
    25  	clk   clock.Clock
    26  	grace time.Duration
    27  }
    28  
    29  // resultEntry is a JSON marshalable exporter result entry.
    30  type resultEntry struct {
    31  	// ID is exported to support marshaling to JSON.
    32  	ID int64 `json:"id"`
    33  
    34  	// Hostname is exported to support marshaling to JSON. Not all queries
    35  	// will fill this field, so it's JSON field tag marks at as
    36  	// omittable.
    37  	Hostname string `json:"hostname,omitempty"`
    38  }
    39  
    40  // reverseHostname converts (reversed) names sourced from the
    41  // registrations table to standard hostnames.
    42  func (r *resultEntry) reverseHostname() {
    43  	r.Hostname = sa.ReverseName(r.Hostname)
    44  }
    45  
    46  // idExporterResults is passed as a selectable 'holder' for the results
    47  // of id-exporter database queries
    48  type idExporterResults []*resultEntry
    49  
    50  // marshalToJSON returns JSON as bytes for all elements of the inner `id`
    51  // slice.
    52  func (i *idExporterResults) marshalToJSON() ([]byte, error) {
    53  	data, err := json.Marshal(i)
    54  	if err != nil {
    55  		return nil, err
    56  	}
    57  	data = append(data, '\n')
    58  	return data, nil
    59  }
    60  
    61  // writeToFile writes the contents of the inner `ids` slice, as JSON, to
    62  // a file
    63  func (i *idExporterResults) writeToFile(outfile string) error {
    64  	data, err := i.marshalToJSON()
    65  	if err != nil {
    66  		return err
    67  	}
    68  	return os.WriteFile(outfile, data, 0644)
    69  }
    70  
    71  // findIDs gathers all registration IDs with unexpired certificates.
    72  func (c idExporter) findIDs(ctx context.Context) (idExporterResults, error) {
    73  	var holder idExporterResults
    74  	_, err := c.dbMap.Select(
    75  		ctx,
    76  		&holder,
    77  		`SELECT DISTINCT r.id
    78  		FROM registrations AS r
    79  			INNER JOIN certificates AS c on c.registrationID = r.id
    80  		WHERE r.contact NOT IN ('[]', 'null')
    81  			AND c.expires >= :expireCutoff;`,
    82  		map[string]interface{}{
    83  			"expireCutoff": c.clk.Now().Add(-c.grace),
    84  		})
    85  	if err != nil {
    86  		c.log.AuditErrf("Error finding IDs: %s", err)
    87  		return nil, err
    88  	}
    89  	return holder, nil
    90  }
    91  
    92  // findIDsWithExampleHostnames gathers all registration IDs with
    93  // unexpired certificates and a corresponding example hostname.
    94  func (c idExporter) findIDsWithExampleHostnames(ctx context.Context) (idExporterResults, error) {
    95  	var holder idExporterResults
    96  	_, err := c.dbMap.Select(
    97  		ctx,
    98  		&holder,
    99  		`SELECT SQL_BIG_RESULT
   100  			cert.registrationID AS id,
   101  			name.reversedName AS hostname
   102  		FROM certificates AS cert
   103  			INNER JOIN issuedNames AS name ON name.serial = cert.serial
   104  		WHERE cert.expires >= :expireCutoff
   105  		GROUP BY cert.registrationID;`,
   106  		map[string]interface{}{
   107  			"expireCutoff": c.clk.Now().Add(-c.grace),
   108  		})
   109  	if err != nil {
   110  		c.log.AuditErrf("Error finding IDs and example hostnames: %s", err)
   111  		return nil, err
   112  	}
   113  
   114  	for _, result := range holder {
   115  		result.reverseHostname()
   116  	}
   117  	return holder, nil
   118  }
   119  
   120  // findIDsForHostnames gathers all registration IDs with unexpired
   121  // certificates for each `hostnames` entry.
   122  func (c idExporter) findIDsForHostnames(ctx context.Context, hostnames []string) (idExporterResults, error) {
   123  	var holder idExporterResults
   124  	for _, hostname := range hostnames {
   125  		// Pass the same list in each time, borp will happily just append to the slice
   126  		// instead of overwriting it each time
   127  		// https://github.com/letsencrypt/borp/blob/c87bd6443d59746a33aca77db34a60cfc344adb2/select.go#L349-L353
   128  		_, err := c.dbMap.Select(
   129  			ctx,
   130  			&holder,
   131  			`SELECT DISTINCT c.registrationID AS id
   132  			FROM certificates AS c
   133  				INNER JOIN issuedNames AS n ON c.serial = n.serial
   134  			WHERE c.expires >= :expireCutoff
   135  				AND n.reversedName = :reversedName;`,
   136  			map[string]interface{}{
   137  				"expireCutoff": c.clk.Now().Add(-c.grace),
   138  				"reversedName": sa.ReverseName(hostname),
   139  			},
   140  		)
   141  		if err != nil {
   142  			if db.IsNoRows(err) {
   143  				continue
   144  			}
   145  			return nil, err
   146  		}
   147  	}
   148  
   149  	return holder, nil
   150  }
   151  
   152  const usageIntro = `
   153  Introduction:
   154  
   155  The ID exporter exists to retrieve the IDs of all registered
   156  users with currently unexpired certificates. This list of registration IDs can
   157  then be given as input to the notification mailer to send bulk notifications.
   158  
   159  The -grace parameter can be used to allow registrations with certificates that
   160  have already expired to be included in the export. The argument is a Go duration
   161  obeying the usual suffix rules (e.g. 24h).
   162  
   163  Registration IDs are favoured over email addresses as the intermediate format in
   164  order to ensure the most up to date contact information is used at the time of
   165  notification. The notification mailer will resolve the ID to email(s) when the
   166  mailing is underway, ensuring we use the correct address if a user has updated
   167  their contact information between the time of export and the time of
   168  notification.
   169  
   170  By default, the ID exporter's output will be JSON of the form:
   171    [
   172      { "id": 1 },
   173      ...
   174      { "id": n }
   175    ]
   176  
   177  Operations that return a hostname will be JSON of the form:
   178    [
   179      { "id": 1, "hostname": "example-1.com" },
   180      ...
   181      { "id": n, "hostname": "example-n.com" }
   182    ]
   183  
   184  Examples:
   185    Export all registration IDs with unexpired certificates to "regs.json":
   186  
   187    id-exporter -config test/config/id-exporter.json -outfile regs.json
   188  
   189    Export all registration IDs with certificates that are unexpired or expired
   190    within the last two days to "regs.json":
   191  
   192    id-exporter -config test/config/id-exporter.json -grace 48h -outfile
   193      "regs.json"
   194  
   195  Required arguments:
   196  - config
   197  - outfile`
   198  
   199  // unmarshalHostnames unmarshals a hostnames file and ensures that the file
   200  // contained at least one entry.
   201  func unmarshalHostnames(filePath string) ([]string, error) {
   202  	file, err := os.Open(filePath)
   203  	if err != nil {
   204  		return nil, err
   205  	}
   206  	defer file.Close()
   207  
   208  	scanner := bufio.NewScanner(file)
   209  	scanner.Split(bufio.ScanLines)
   210  
   211  	var hostnames []string
   212  	for scanner.Scan() {
   213  		line := scanner.Text()
   214  		if strings.Contains(line, " ") {
   215  			return nil, fmt.Errorf(
   216  				"line: %q contains more than one entry, entries must be separated by newlines", line)
   217  		}
   218  		hostnames = append(hostnames, line)
   219  	}
   220  
   221  	if len(hostnames) == 0 {
   222  		return nil, errors.New("provided file contains 0 hostnames")
   223  	}
   224  	return hostnames, nil
   225  }
   226  
   227  type Config struct {
   228  	ContactExporter struct {
   229  		DB cmd.DBConfig
   230  		cmd.PasswordConfig
   231  		Features map[string]bool
   232  	}
   233  }
   234  
   235  func main() {
   236  	outFile := flag.String("outfile", "", "File to output results JSON to.")
   237  	grace := flag.Duration("grace", 2*24*time.Hour, "Include results with certificates that expired in < grace ago.")
   238  	hostnamesFile := flag.String(
   239  		"hostnames", "", "Only include results with unexpired certificates that contain hostnames\nlisted (newline separated) in this file.")
   240  	withExampleHostnames := flag.Bool(
   241  		"with-example-hostnames", false, "Include an example hostname for each registration ID with an unexpired certificate.")
   242  	configFile := flag.String("config", "", "File containing a JSON config.")
   243  
   244  	flag.Usage = func() {
   245  		fmt.Fprintf(os.Stderr, "%s\n\n", usageIntro)
   246  		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
   247  		flag.PrintDefaults()
   248  	}
   249  
   250  	// Parse flags and check required.
   251  	flag.Parse()
   252  	if *outFile == "" || *configFile == "" {
   253  		flag.Usage()
   254  		os.Exit(1)
   255  	}
   256  
   257  	log := cmd.NewLogger(cmd.SyslogConfig{StdoutLevel: 7})
   258  	log.Info(cmd.VersionString())
   259  
   260  	// Load configuration file.
   261  	configData, err := os.ReadFile(*configFile)
   262  	cmd.FailOnError(err, fmt.Sprintf("Reading %q", *configFile))
   263  
   264  	// Unmarshal JSON config file.
   265  	var cfg Config
   266  	err = json.Unmarshal(configData, &cfg)
   267  	cmd.FailOnError(err, "Unmarshaling config")
   268  
   269  	err = features.Set(cfg.ContactExporter.Features)
   270  	cmd.FailOnError(err, "Failed to set feature flags")
   271  
   272  	dbMap, err := sa.InitWrappedDb(cfg.ContactExporter.DB, nil, log)
   273  	cmd.FailOnError(err, "While initializing dbMap")
   274  
   275  	exporter := idExporter{
   276  		log:   log,
   277  		dbMap: dbMap,
   278  		clk:   cmd.Clock(),
   279  		grace: *grace,
   280  	}
   281  
   282  	var results idExporterResults
   283  	if *hostnamesFile != "" {
   284  		hostnames, err := unmarshalHostnames(*hostnamesFile)
   285  		cmd.FailOnError(err, "Problem unmarshalling hostnames")
   286  
   287  		results, err = exporter.findIDsForHostnames(context.TODO(), hostnames)
   288  		cmd.FailOnError(err, "Could not find IDs for hostnames")
   289  
   290  	} else if *withExampleHostnames {
   291  		results, err = exporter.findIDsWithExampleHostnames(context.TODO())
   292  		cmd.FailOnError(err, "Could not find IDs with hostnames")
   293  
   294  	} else {
   295  		results, err = exporter.findIDs(context.TODO())
   296  		cmd.FailOnError(err, "Could not find IDs")
   297  	}
   298  
   299  	err = results.writeToFile(*outFile)
   300  	cmd.FailOnError(err, fmt.Sprintf("Could not write result to outfile %q", *outFile))
   301  }
   302  
   303  func init() {
   304  	cmd.RegisterCommand("id-exporter", main, &cmd.ConfigValidator{Config: &Config{}})
   305  }
   306
View as plain text