Source file src/cloud.google.com/go/logging/logging.go

Documentation: cloud.google.com/go/logging

     1  // Copyright 2016 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  // API/gRPC features intentionally missing from this client:
    16  // - You cannot have the server pick the time of the entry. This client
    17  //   always sends a time.
    18  // - There is no way to provide a protocol buffer payload.
    19  // - No support for the "partial success" feature when writing log entries.
    20  
    21  // TODO(jba): test whether forward-slash characters in the log ID must be URL-encoded.
    22  // These features are missing now, but will likely be added:
    23  // - There is no way to specify CallOptions.
    24  
    25  package logging
    26  
    27  import (
    28  	"bytes"
    29  	"context"
    30  	"encoding/json"
    31  	"errors"
    32  	"fmt"
    33  	"io"
    34  	"log"
    35  	"net/http"
    36  	"regexp"
    37  	"runtime"
    38  	"strconv"
    39  	"strings"
    40  	"sync"
    41  	"time"
    42  	"unicode/utf8"
    43  
    44  	vkit "cloud.google.com/go/logging/apiv2"
    45  	logpb "cloud.google.com/go/logging/apiv2/loggingpb"
    46  	"cloud.google.com/go/logging/internal"
    47  	gax "github.com/googleapis/gax-go/v2"
    48  	"google.golang.org/api/option"
    49  	"google.golang.org/api/support/bundler"
    50  	mrpb "google.golang.org/genproto/googleapis/api/monitoredres"
    51  	logtypepb "google.golang.org/genproto/googleapis/logging/type"
    52  	"google.golang.org/grpc/codes"
    53  	"google.golang.org/grpc/status"
    54  	"google.golang.org/protobuf/encoding/protojson"
    55  	"google.golang.org/protobuf/proto"
    56  	"google.golang.org/protobuf/types/known/anypb"
    57  	"google.golang.org/protobuf/types/known/durationpb"
    58  	structpb "google.golang.org/protobuf/types/known/structpb"
    59  	"google.golang.org/protobuf/types/known/timestamppb"
    60  )
    61  
    62  const (
    63  	// ReadScope is the scope for reading from the logging service.
    64  	ReadScope = "https://www.googleapis.com/auth/logging.read"
    65  
    66  	// WriteScope is the scope for writing to the logging service.
    67  	WriteScope = "https://www.googleapis.com/auth/logging.write"
    68  
    69  	// AdminScope is the scope for administrative actions on the logging service.
    70  	AdminScope = "https://www.googleapis.com/auth/logging.admin"
    71  )
    72  
    73  const (
    74  	// defaultErrorCapacity is the capacity of the channel used to deliver
    75  	// errors to the OnError function.
    76  	defaultErrorCapacity = 10
    77  
    78  	// DefaultDelayThreshold is the default value for the DelayThreshold LoggerOption.
    79  	DefaultDelayThreshold = time.Second
    80  
    81  	// DefaultEntryCountThreshold is the default value for the EntryCountThreshold LoggerOption.
    82  	DefaultEntryCountThreshold = 1000
    83  
    84  	// DefaultEntryByteThreshold is the default value for the EntryByteThreshold LoggerOption.
    85  	DefaultEntryByteThreshold = 1 << 23 // 8MiB
    86  
    87  	// DefaultBundleByteLimit is the default value for the BundleByteLimit LoggerOption.
    88  	DefaultBundleByteLimit = 9437184 // 9.5 MiB
    89  
    90  	// DefaultBufferedByteLimit is the default value for the BufferedByteLimit LoggerOption.
    91  	DefaultBufferedByteLimit = 1 << 30 // 1GiB
    92  
    93  	// defaultWriteTimeout is the timeout for the underlying write API calls. As
    94  	// write API calls are not idempotent, they are not retried on timeout. This
    95  	// timeout is to allow clients to degrade gracefully if underlying logging
    96  	// service is temporarily impaired for some reason.
    97  	defaultWriteTimeout = 10 * time.Minute
    98  
    99  	// Part of the error message when the payload contains invalid UTF-8 characters.
   100  	utfErrorString = "string field contains invalid UTF-8"
   101  
   102  	// DetectProjectID is a sentinel value that instructs NewClient to detect the
   103  	// project ID. It is given in place of the projectID argument. NewClient will
   104  	// use the project ID from the given credentials or the default credentials
   105  	// (https://developers.google.com/accounts/docs/application-default-credentials)
   106  	// if no credentials were provided. When providing credentials, not all
   107  	// options will allow NewClient to extract the project ID. Specifically a JWT
   108  	// does not have the project ID encoded.
   109  	DetectProjectID = "*detect-project-id*"
   110  )
   111  
   112  var (
   113  	// ErrRedirectProtoPayloadNotSupported is returned when Logger is configured to redirect output and
   114  	// tries to redirect logs with protobuf payload.
   115  	ErrRedirectProtoPayloadNotSupported = errors.New("printEntryToStdout: cannot find valid payload")
   116  
   117  	// For testing:
   118  	now                    = time.Now
   119  	toLogEntryInternal     = toLogEntryInternalImpl
   120  	detectResourceInternal = detectResource
   121  
   122  	// ErrOverflow signals that the number of buffered entries for a Logger
   123  	// exceeds its BufferLimit.
   124  	ErrOverflow = bundler.ErrOverflow
   125  
   126  	// ErrOversizedEntry signals that an entry's size exceeds the maximum number of
   127  	// bytes that will be sent in a single call to the logging service.
   128  	ErrOversizedEntry = bundler.ErrOversizedItem
   129  )
   130  
   131  // Client is a Logging client. A Client is associated with a single Cloud project.
   132  type Client struct {
   133  	client  *vkit.Client   // client for the logging service
   134  	parent  string         // e.g. "projects/proj-id"
   135  	errc    chan error     // should be buffered to minimize dropped errors
   136  	donec   chan struct{}  // closed on Client.Close to close Logger bundlers
   137  	loggers sync.WaitGroup // so we can wait for loggers to close
   138  	closed  bool
   139  
   140  	mu      sync.Mutex
   141  	nErrs   int   // number of errors we saw
   142  	lastErr error // last error we saw
   143  
   144  	// OnError is called when an error occurs in a call to Log or Flush. The
   145  	// error may be due to an invalid Entry, an overflow because BufferLimit
   146  	// was reached (in which case the error will be ErrOverflow) or an error
   147  	// communicating with the logging service. OnError is called with errors
   148  	// from all Loggers. It is never called concurrently. OnError is expected
   149  	// to return quickly; if errors occur while OnError is running, some may
   150  	// not be reported. The default behavior is to call log.Printf.
   151  	//
   152  	// This field should be set only once, before any method of Client is called.
   153  	OnError func(err error)
   154  }
   155  
   156  // NewClient returns a new logging client associated with the provided parent.
   157  // A parent can take any of the following forms:
   158  //
   159  //	projects/PROJECT_ID
   160  //	folders/FOLDER_ID
   161  //	billingAccounts/ACCOUNT_ID
   162  //	organizations/ORG_ID
   163  //
   164  // For backwards compatibility, a string with no '/' is also allowed and is interpreted
   165  // as a project ID.
   166  //
   167  // If logging.DetectProjectId is provided as the parent, the parent will be interpreted as a project
   168  // ID, and its value will be inferred from the environment.
   169  //
   170  // By default NewClient uses WriteScope. To use a different scope, call
   171  // NewClient using a WithScopes option (see https://godoc.org/google.golang.org/api/option#WithScopes).
   172  func NewClient(ctx context.Context, parent string, opts ...option.ClientOption) (*Client, error) {
   173  	parent, err := makeParent(parent)
   174  	if err != nil {
   175  		return nil, err
   176  	}
   177  	opts = append([]option.ClientOption{
   178  		option.WithScopes(WriteScope),
   179  	}, opts...)
   180  	c, err := vkit.NewClient(ctx, opts...)
   181  	if err != nil {
   182  		return nil, err
   183  	}
   184  	c.SetGoogleClientInfo("gccl", internal.Version)
   185  	client := &Client{
   186  		client:  c,
   187  		parent:  parent,
   188  		errc:    make(chan error, defaultErrorCapacity), // create a small buffer for errors
   189  		donec:   make(chan struct{}),
   190  		OnError: func(e error) { log.Printf("logging client: %v", e) },
   191  	}
   192  	// Call the user's function synchronously, to make life easier for them.
   193  	go func() {
   194  		for err := range client.errc {
   195  			// This reference to OnError is memory-safe if the user sets OnError before
   196  			// calling any client methods. The reference happens before the first read from
   197  			// client.errc, which happens before the first write to client.errc, which
   198  			// happens before any call, which happens before the user sets OnError.
   199  			if fn := client.OnError; fn != nil {
   200  				fn(err)
   201  			} else {
   202  				log.Printf("logging (parent %q): %v", parent, err)
   203  			}
   204  		}
   205  	}()
   206  	return client, nil
   207  }
   208  
   209  func makeParent(parent string) (string, error) {
   210  	if !strings.ContainsRune(parent, '/') {
   211  		if parent == DetectProjectID {
   212  			resource := detectResourceInternal()
   213  			if resource == nil {
   214  				return parent, fmt.Errorf("could not determine project ID from environment")
   215  			}
   216  			parent = resource.Labels["project_id"]
   217  		}
   218  		return "projects/" + parent, nil
   219  	}
   220  	prefix := strings.Split(parent, "/")[0]
   221  	if prefix != "projects" && prefix != "folders" && prefix != "billingAccounts" && prefix != "organizations" {
   222  		return parent, fmt.Errorf("parent parameter must start with 'projects/' 'folders/' 'billingAccounts/' or 'organizations/'")
   223  	}
   224  	return parent, nil
   225  }
   226  
   227  // Ping reports whether the client's connection to the logging service and the
   228  // authentication configuration are valid. To accomplish this, Ping writes a
   229  // log entry "ping" to a log named "ping".
   230  func (c *Client) Ping(ctx context.Context) error {
   231  	unixZeroTimestamp := timestamppb.New(time.Unix(0, 0))
   232  	ent := &logpb.LogEntry{
   233  		Payload:   &logpb.LogEntry_TextPayload{TextPayload: "ping"},
   234  		Timestamp: unixZeroTimestamp, // Identical timestamps and insert IDs are both
   235  		InsertId:  "ping",            // necessary for the service to dedup these entries.
   236  	}
   237  	_, err := c.client.WriteLogEntries(ctx, &logpb.WriteLogEntriesRequest{
   238  		LogName:  internal.LogPath(c.parent, "ping"),
   239  		Resource: monitoredResource(c.parent),
   240  		Entries:  []*logpb.LogEntry{ent},
   241  	})
   242  	return err
   243  }
   244  
   245  // error puts the error on the client's error channel
   246  // without blocking, and records summary error info.
   247  func (c *Client) error(err error) {
   248  	select {
   249  	case c.errc <- err:
   250  	default:
   251  	}
   252  	c.mu.Lock()
   253  	c.lastErr = err
   254  	c.nErrs++
   255  	c.mu.Unlock()
   256  }
   257  
   258  func (c *Client) extractErrorInfo() error {
   259  	var err error
   260  	c.mu.Lock()
   261  	if c.lastErr != nil {
   262  		err = fmt.Errorf("saw %d errors; last: %w", c.nErrs, c.lastErr)
   263  		c.nErrs = 0
   264  		c.lastErr = nil
   265  	}
   266  	c.mu.Unlock()
   267  	return err
   268  }
   269  
   270  // A Logger is used to write log messages to a single log. It can be configured
   271  // with a log ID, common monitored resource, and a set of common labels.
   272  type Logger struct {
   273  	client     *Client
   274  	logName    string // "projects/{projectID}/logs/{logID}"
   275  	stdLoggers map[Severity]*log.Logger
   276  	bundler    *bundler.Bundler
   277  
   278  	// Options
   279  	commonResource         *mrpb.MonitoredResource
   280  	commonLabels           map[string]string
   281  	ctxFunc                func() (context.Context, func())
   282  	populateSourceLocation int
   283  	partialSuccess         bool
   284  	redirectOutputWriter   io.Writer
   285  }
   286  
   287  type loggerRetryer struct {
   288  	defaultRetryer gax.Retryer
   289  }
   290  
   291  func newLoggerRetryer() gax.Retryer {
   292  	// Copied from CallOptions.WriteLogEntries in apiv2/logging_client.go.
   293  	d := gax.OnCodes([]codes.Code{
   294  		codes.DeadlineExceeded,
   295  		codes.Internal,
   296  		codes.Unavailable,
   297  	}, gax.Backoff{
   298  		Initial:    100 * time.Millisecond,
   299  		Max:        60000 * time.Millisecond,
   300  		Multiplier: 1.30,
   301  	})
   302  
   303  	r := &loggerRetryer{defaultRetryer: d}
   304  	return r
   305  }
   306  
   307  func (r *loggerRetryer) Retry(err error) (pause time.Duration, shouldRetry bool) {
   308  	s, ok := status.FromError(err)
   309  	if !ok {
   310  		return r.defaultRetryer.Retry(err)
   311  	}
   312  	if strings.Contains(s.Message(), utfErrorString) {
   313  		return 0, false
   314  	}
   315  	return r.defaultRetryer.Retry(err)
   316  }
   317  
   318  // Logger returns a Logger that will write entries with the given log ID, such as
   319  // "syslog". A log ID must be less than 512 characters long and can only
   320  // include the following characters: upper and lower case alphanumeric
   321  // characters: [A-Za-z0-9]; and punctuation characters: forward-slash,
   322  // underscore, hyphen, and period.
   323  func (c *Client) Logger(logID string, opts ...LoggerOption) *Logger {
   324  	r := detectResourceInternal()
   325  	if r == nil {
   326  		r = monitoredResource(c.parent)
   327  	}
   328  	l := &Logger{
   329  		client:                 c,
   330  		logName:                internal.LogPath(c.parent, logID),
   331  		commonResource:         r,
   332  		ctxFunc:                func() (context.Context, func()) { return context.Background(), nil },
   333  		populateSourceLocation: DoNotPopulateSourceLocation,
   334  		partialSuccess:         false,
   335  		redirectOutputWriter:   nil,
   336  	}
   337  	l.bundler = bundler.NewBundler(&logpb.LogEntry{}, func(entries interface{}) {
   338  		l.writeLogEntries(entries.([]*logpb.LogEntry))
   339  	})
   340  	l.bundler.DelayThreshold = DefaultDelayThreshold
   341  	l.bundler.BundleCountThreshold = DefaultEntryCountThreshold
   342  	l.bundler.BundleByteThreshold = DefaultEntryByteThreshold
   343  	l.bundler.BundleByteLimit = DefaultBundleByteLimit
   344  	l.bundler.BufferedByteLimit = DefaultBufferedByteLimit
   345  	for _, opt := range opts {
   346  		opt.set(l)
   347  	}
   348  	l.stdLoggers = map[Severity]*log.Logger{}
   349  	for s := range severityName {
   350  		e := Entry{Severity: s}
   351  		l.stdLoggers[s] = log.New(templateEntryWriter{l, &e}, "", 0)
   352  	}
   353  
   354  	c.loggers.Add(1)
   355  	// Start a goroutine that cleans up the bundler, its channel
   356  	// and the writer goroutines when the client is closed.
   357  	go func() {
   358  		defer c.loggers.Done()
   359  		<-c.donec
   360  		l.bundler.Flush()
   361  	}()
   362  	return l
   363  }
   364  
   365  type templateEntryWriter struct {
   366  	l        *Logger
   367  	template *Entry
   368  }
   369  
   370  func (w templateEntryWriter) Write(p []byte) (n int, err error) {
   371  	e := *w.template
   372  	e.Payload = string(p)
   373  	// The second argument to logInternal() is how many frames to skip
   374  	// from the call stack when determining the source location. In the
   375  	// current implementation of log.Logger (i.e. Go's logging library)
   376  	// the Write() method is called 2 calls deep so we need to skip 3
   377  	// frames to account for the call to logInternal() itself.
   378  	w.l.logInternal(e, 3)
   379  	return len(p), nil
   380  }
   381  
   382  // Close waits for all opened loggers to be flushed and closes the client.
   383  func (c *Client) Close() error {
   384  	if c.closed {
   385  		return nil
   386  	}
   387  	close(c.donec)   // close Logger bundlers
   388  	c.loggers.Wait() // wait for all bundlers to flush and close
   389  	// Now there can be no more errors.
   390  	close(c.errc) // terminate error goroutine
   391  	// Prefer errors arising from logging to the error returned from Close.
   392  	err := c.extractErrorInfo()
   393  	err2 := c.client.Close()
   394  	if err == nil {
   395  		err = err2
   396  	}
   397  	c.closed = true
   398  	return err
   399  }
   400  
   401  // Severity is the severity of the event described in a log entry. These
   402  // guideline severity levels are ordered, with numerically smaller levels
   403  // treated as less severe than numerically larger levels.
   404  type Severity int
   405  
   406  const (
   407  	// Default means the log entry has no assigned severity level.
   408  	Default = Severity(logtypepb.LogSeverity_DEFAULT)
   409  	// Debug means debug or trace information.
   410  	Debug = Severity(logtypepb.LogSeverity_DEBUG)
   411  	// Info means routine information, such as ongoing status or performance.
   412  	Info = Severity(logtypepb.LogSeverity_INFO)
   413  	// Notice means normal but significant events, such as start up, shut down, or configuration.
   414  	Notice = Severity(logtypepb.LogSeverity_NOTICE)
   415  	// Warning means events that might cause problems.
   416  	Warning = Severity(logtypepb.LogSeverity_WARNING)
   417  	// Error means events that are likely to cause problems.
   418  	Error = Severity(logtypepb.LogSeverity_ERROR)
   419  	// Critical means events that cause more severe problems or brief outages.
   420  	Critical = Severity(logtypepb.LogSeverity_CRITICAL)
   421  	// Alert means a person must take an action immediately.
   422  	Alert = Severity(logtypepb.LogSeverity_ALERT)
   423  	// Emergency means one or more systems are unusable.
   424  	Emergency = Severity(logtypepb.LogSeverity_EMERGENCY)
   425  )
   426  
   427  var severityName = map[Severity]string{
   428  	Default:   "Default",
   429  	Debug:     "Debug",
   430  	Info:      "Info",
   431  	Notice:    "Notice",
   432  	Warning:   "Warning",
   433  	Error:     "Error",
   434  	Critical:  "Critical",
   435  	Alert:     "Alert",
   436  	Emergency: "Emergency",
   437  }
   438  
   439  // String converts a severity level to a string.
   440  func (v Severity) String() string {
   441  	// same as proto.EnumName
   442  	s, ok := severityName[v]
   443  	if ok {
   444  		return s
   445  	}
   446  	return strconv.Itoa(int(v))
   447  }
   448  
   449  // UnmarshalJSON turns a string representation of severity into the type
   450  // Severity.
   451  func (v *Severity) UnmarshalJSON(data []byte) error {
   452  	var s string
   453  	var i int
   454  	if strErr := json.Unmarshal(data, &s); strErr == nil {
   455  		*v = ParseSeverity(s)
   456  	} else if intErr := json.Unmarshal(data, &i); intErr == nil {
   457  		*v = Severity(i)
   458  	} else {
   459  		return fmt.Errorf("%v; %v", strErr, intErr)
   460  	}
   461  	return nil
   462  }
   463  
   464  // ParseSeverity returns the Severity whose name equals s, ignoring case. It
   465  // returns Default if no Severity matches.
   466  func ParseSeverity(s string) Severity {
   467  	sl := strings.ToLower(s)
   468  	for sev, name := range severityName {
   469  		if strings.ToLower(name) == sl {
   470  			return sev
   471  		}
   472  	}
   473  	return Default
   474  }
   475  
   476  // Entry is a log entry.
   477  // See https://cloud.google.com/logging/docs/view/logs_index for more about entries.
   478  type Entry struct {
   479  	// Timestamp is the time of the entry. If zero, the current time is used.
   480  	Timestamp time.Time
   481  
   482  	// Severity is the entry's severity level.
   483  	// The zero value is Default.
   484  	Severity Severity
   485  
   486  	// Payload must be either a string, or something that marshals via the
   487  	// encoding/json package to a JSON object (and not any other type of JSON value).
   488  	Payload interface{}
   489  
   490  	// Labels optionally specifies key/value labels for the log entry.
   491  	// The Logger.Log method takes ownership of this map. See Logger.CommonLabels
   492  	// for more about labels.
   493  	Labels map[string]string
   494  
   495  	// InsertID is a unique ID for the log entry. If you provide this field,
   496  	// the logging service considers other log entries in the same log with the
   497  	// same ID as duplicates which can be removed. If omitted, the logging
   498  	// service will generate a unique ID for this log entry. Note that because
   499  	// this client retries RPCs automatically, it is possible (though unlikely)
   500  	// that an Entry without an InsertID will be written more than once.
   501  	InsertID string
   502  
   503  	// HTTPRequest optionally specifies metadata about the HTTP request
   504  	// associated with this log entry, if applicable. It is optional.
   505  	HTTPRequest *HTTPRequest
   506  
   507  	// Operation optionally provides information about an operation associated
   508  	// with the log entry, if applicable.
   509  	Operation *logpb.LogEntryOperation
   510  
   511  	// LogName is the full log name, in the form
   512  	// "projects/{ProjectID}/logs/{LogID}". It is set by the client when
   513  	// reading entries. It is an error to set it when writing entries.
   514  	LogName string
   515  
   516  	// Resource is the monitored resource associated with the entry.
   517  	Resource *mrpb.MonitoredResource
   518  
   519  	// Trace is the resource name of the trace associated with the log entry,
   520  	// if any. If it contains a relative resource name, the name is assumed to
   521  	// be relative to //tracing.googleapis.com.
   522  	Trace string
   523  
   524  	// ID of the span within the trace associated with the log entry.
   525  	// The ID is a 16-character hexadecimal encoding of an 8-byte array.
   526  	SpanID string
   527  
   528  	// If set, symbolizes that this request was sampled.
   529  	TraceSampled bool
   530  
   531  	// Optional. Source code location information associated with the log entry,
   532  	// if any.
   533  	SourceLocation *logpb.LogEntrySourceLocation
   534  }
   535  
   536  // HTTPRequest contains an http.Request as well as additional
   537  // information about the request and its response.
   538  type HTTPRequest struct {
   539  	// Request is the http.Request passed to the handler.
   540  	Request *http.Request
   541  
   542  	// RequestSize is the size of the HTTP request message in bytes, including
   543  	// the request headers and the request body.
   544  	RequestSize int64
   545  
   546  	// Status is the response code indicating the status of the response.
   547  	// Examples: 200, 404.
   548  	Status int
   549  
   550  	// ResponseSize is the size of the HTTP response message sent back to the client, in bytes,
   551  	// including the response headers and the response body.
   552  	ResponseSize int64
   553  
   554  	// Latency is the request processing latency on the server, from the time the request was
   555  	// received until the response was sent.
   556  	Latency time.Duration
   557  
   558  	// LocalIP is the IP address (IPv4 or IPv6) of the origin server that the request
   559  	// was sent to.
   560  	LocalIP string
   561  
   562  	// RemoteIP is the IP address (IPv4 or IPv6) of the client that issued the
   563  	// HTTP request. Examples: "192.168.1.1", "FE80::0202:B3FF:FE1E:8329".
   564  	RemoteIP string
   565  
   566  	// CacheHit reports whether an entity was served from cache (with or without
   567  	// validation).
   568  	CacheHit bool
   569  
   570  	// CacheValidatedWithOriginServer reports whether the response was
   571  	// validated with the origin server before being served from cache. This
   572  	// field is only meaningful if CacheHit is true.
   573  	CacheValidatedWithOriginServer bool
   574  
   575  	// CacheFillBytes is the number of HTTP response bytes inserted into cache. Set only when a cache fill was attempted.
   576  	CacheFillBytes int64
   577  
   578  	// CacheLookup tells whether or not a cache lookup was attempted.
   579  	CacheLookup bool
   580  }
   581  
   582  func fromHTTPRequest(r *HTTPRequest) (*logtypepb.HttpRequest, error) {
   583  	if r == nil {
   584  		return nil, nil
   585  	}
   586  	if r.Request == nil {
   587  		return nil, errors.New("logging: HTTPRequest must have a non-nil Request")
   588  	}
   589  	u := *r.Request.URL
   590  	u.Fragment = ""
   591  	pb := &logtypepb.HttpRequest{
   592  		RequestMethod:                  r.Request.Method,
   593  		RequestUrl:                     fixUTF8(u.String()),
   594  		RequestSize:                    r.RequestSize,
   595  		Status:                         int32(r.Status),
   596  		ResponseSize:                   r.ResponseSize,
   597  		UserAgent:                      r.Request.UserAgent(),
   598  		ServerIp:                       r.LocalIP,
   599  		RemoteIp:                       r.RemoteIP, // TODO(jba): attempt to parse http.Request.RemoteAddr?
   600  		Referer:                        r.Request.Referer(),
   601  		CacheHit:                       r.CacheHit,
   602  		CacheValidatedWithOriginServer: r.CacheValidatedWithOriginServer,
   603  		Protocol:                       r.Request.Proto,
   604  		CacheFillBytes:                 r.CacheFillBytes,
   605  		CacheLookup:                    r.CacheLookup,
   606  	}
   607  	if r.Latency != 0 {
   608  		pb.Latency = durationpb.New(r.Latency)
   609  	}
   610  	return pb, nil
   611  }
   612  
   613  // fixUTF8 is a helper that fixes an invalid UTF-8 string by replacing
   614  // invalid UTF-8 runes with the Unicode replacement character (U+FFFD).
   615  // See Issue https://github.com/googleapis/google-cloud-go/issues/1383.
   616  func fixUTF8(s string) string {
   617  	if utf8.ValidString(s) {
   618  		return s
   619  	}
   620  
   621  	// Otherwise time to build the sequence.
   622  	buf := new(bytes.Buffer)
   623  	buf.Grow(len(s))
   624  	for _, r := range s {
   625  		if utf8.ValidRune(r) {
   626  			buf.WriteRune(r)
   627  		} else {
   628  			buf.WriteRune('\uFFFD')
   629  		}
   630  	}
   631  	return buf.String()
   632  }
   633  
   634  // toProtoStruct converts v, which must marshal into a JSON object,
   635  // into a Google Struct proto.
   636  func toProtoStruct(v interface{}) (*structpb.Struct, error) {
   637  	// Fast path: if v is already a *structpb.Struct, nothing to do.
   638  	if s, ok := v.(*structpb.Struct); ok {
   639  		return s, nil
   640  	}
   641  	// v is a Go value that supports JSON marshalling. We want a Struct
   642  	// protobuf. Some day we may have a more direct way to get there, but right
   643  	// now the only way is to marshal the Go value to JSON, unmarshal into a
   644  	// map, and then build the Struct proto from the map.
   645  	var jb []byte
   646  	var err error
   647  	if raw, ok := v.(json.RawMessage); ok { // needed for Go 1.7 and below
   648  		jb = []byte(raw)
   649  	} else {
   650  		jb, err = json.Marshal(v)
   651  		if err != nil {
   652  			return nil, fmt.Errorf("logging: json.Marshal: %w", err)
   653  		}
   654  	}
   655  	var m map[string]interface{}
   656  	err = json.Unmarshal(jb, &m)
   657  	if err != nil {
   658  		return nil, fmt.Errorf("logging: json.Unmarshal: %w", err)
   659  	}
   660  	return jsonMapToProtoStruct(m), nil
   661  }
   662  
   663  func jsonMapToProtoStruct(m map[string]interface{}) *structpb.Struct {
   664  	fields := map[string]*structpb.Value{}
   665  	for k, v := range m {
   666  		fields[k] = jsonValueToStructValue(v)
   667  	}
   668  	return &structpb.Struct{Fields: fields}
   669  }
   670  
   671  func jsonValueToStructValue(v interface{}) *structpb.Value {
   672  	switch x := v.(type) {
   673  	case bool:
   674  		return &structpb.Value{Kind: &structpb.Value_BoolValue{BoolValue: x}}
   675  	case float64:
   676  		return &structpb.Value{Kind: &structpb.Value_NumberValue{NumberValue: x}}
   677  	case string:
   678  		return &structpb.Value{Kind: &structpb.Value_StringValue{StringValue: x}}
   679  	case nil:
   680  		return &structpb.Value{Kind: &structpb.Value_NullValue{}}
   681  	case map[string]interface{}:
   682  		return &structpb.Value{Kind: &structpb.Value_StructValue{StructValue: jsonMapToProtoStruct(x)}}
   683  	case []interface{}:
   684  		var vals []*structpb.Value
   685  		for _, e := range x {
   686  			vals = append(vals, jsonValueToStructValue(e))
   687  		}
   688  		return &structpb.Value{Kind: &structpb.Value_ListValue{ListValue: &structpb.ListValue{Values: vals}}}
   689  	default:
   690  		return &structpb.Value{Kind: &structpb.Value_NullValue{}}
   691  	}
   692  }
   693  
   694  // LogSync logs the Entry synchronously without any buffering. Because LogSync is slow
   695  // and will block, it is intended primarily for debugging or critical errors.
   696  // Prefer Log for most uses.
   697  func (l *Logger) LogSync(ctx context.Context, e Entry) error {
   698  	ent, err := toLogEntryInternal(e, l, l.client.parent, 1)
   699  	if err != nil {
   700  		return err
   701  	}
   702  	entries, hasInstrumentation := l.instrumentLogs([]*logpb.LogEntry{ent})
   703  	if l.redirectOutputWriter != nil {
   704  		for _, ent = range entries {
   705  			err = serializeEntryToWriter(ent, l.redirectOutputWriter)
   706  			if err != nil {
   707  				break
   708  			}
   709  		}
   710  		return err
   711  	}
   712  	_, err = l.client.client.WriteLogEntries(ctx, &logpb.WriteLogEntriesRequest{
   713  		LogName:        l.logName,
   714  		Resource:       l.commonResource,
   715  		Labels:         l.commonLabels,
   716  		Entries:        entries,
   717  		PartialSuccess: l.partialSuccess || hasInstrumentation,
   718  	})
   719  	return err
   720  }
   721  
   722  // Log buffers the Entry for output to the logging service. It never blocks.
   723  func (l *Logger) Log(e Entry) {
   724  	l.logInternal(e, 1)
   725  }
   726  
   727  func (l *Logger) logInternal(e Entry, skipLevels int) {
   728  	ent, err := toLogEntryInternal(e, l, l.client.parent, skipLevels+1)
   729  	if err != nil {
   730  		l.client.error(err)
   731  		return
   732  	}
   733  
   734  	entries, _ := l.instrumentLogs([]*logpb.LogEntry{ent})
   735  	if l.redirectOutputWriter != nil {
   736  		for _, ent = range entries {
   737  			err = serializeEntryToWriter(ent, l.redirectOutputWriter)
   738  			if err != nil {
   739  				l.client.error(err)
   740  			}
   741  		}
   742  		return
   743  	}
   744  	for _, ent = range entries {
   745  		if err := l.bundler.Add(ent, proto.Size(ent)); err != nil {
   746  			l.client.error(err)
   747  		}
   748  	}
   749  }
   750  
   751  // Flush blocks until all currently buffered log entries are sent.
   752  //
   753  // If any errors occurred since the last call to Flush from any Logger, or the
   754  // creation of the client if this is the first call, then Flush returns a non-nil
   755  // error with summary information about the errors. This information is unlikely to
   756  // be actionable. For more accurate error reporting, set Client.OnError.
   757  func (l *Logger) Flush() error {
   758  	l.bundler.Flush()
   759  	return l.client.extractErrorInfo()
   760  }
   761  
   762  func (l *Logger) writeLogEntries(entries []*logpb.LogEntry) {
   763  	partialSuccess := l.partialSuccess
   764  	if len(entries) > 1 {
   765  		partialSuccess = partialSuccess || hasInstrumentation(entries)
   766  	}
   767  	req := &logpb.WriteLogEntriesRequest{
   768  		LogName:        l.logName,
   769  		Resource:       l.commonResource,
   770  		Labels:         l.commonLabels,
   771  		Entries:        entries,
   772  		PartialSuccess: partialSuccess,
   773  	}
   774  	ctx, afterCall := l.ctxFunc()
   775  	ctx, cancel := context.WithTimeout(ctx, defaultWriteTimeout)
   776  	defer cancel()
   777  
   778  	_, err := l.client.client.WriteLogEntries(ctx, req, gax.WithRetry(newLoggerRetryer))
   779  	if err != nil {
   780  		l.client.error(err)
   781  	}
   782  	if afterCall != nil {
   783  		afterCall()
   784  	}
   785  }
   786  
   787  // StandardLogger returns a *log.Logger for the provided severity.
   788  //
   789  // This method is cheap. A single log.Logger is pre-allocated for each
   790  // severity level in each Logger. Callers may mutate the returned log.Logger
   791  // (for example by calling SetFlags or SetPrefix).
   792  func (l *Logger) StandardLogger(s Severity) *log.Logger { return l.stdLoggers[s] }
   793  
   794  // StandardLoggerFromTemplate returns a Go Standard Logging API *log.Logger.
   795  //
   796  // The returned logger emits logs using logging.(*Logger).Log() with an entry
   797  // constructed from the provided template Entry struct.
   798  //
   799  // The caller is responsible for ensuring that the template Entry struct
   800  // does not change during the the lifetime of the returned *log.Logger.
   801  //
   802  // Prefer (*Logger).StandardLogger() which is more efficient if the template
   803  // only sets Severity.
   804  func (l *Logger) StandardLoggerFromTemplate(template *Entry) *log.Logger {
   805  	return log.New(templateEntryWriter{l, template}, "", 0)
   806  }
   807  
   808  func populateTraceInfo(e *Entry, req *http.Request) bool {
   809  	if req == nil {
   810  		if e.HTTPRequest != nil && e.HTTPRequest.Request != nil {
   811  			req = e.HTTPRequest.Request
   812  		} else {
   813  			return false
   814  		}
   815  	}
   816  	header := req.Header.Get("Traceparent")
   817  	if header != "" {
   818  		// do not use traceSampled flag defined by traceparent because
   819  		// flag's definition differs from expected by Cloud Tracing
   820  		traceID, spanID, _ := deconstructTraceParent(header)
   821  		if traceID != "" {
   822  			e.Trace = traceID
   823  			e.SpanID = spanID
   824  			return true
   825  		}
   826  	}
   827  	header = req.Header.Get("X-Cloud-Trace-Context")
   828  	if header != "" {
   829  		traceID, spanID, traceSampled := deconstructXCloudTraceContext(header)
   830  		if traceID != "" {
   831  			e.Trace = traceID
   832  			e.SpanID = spanID
   833  			// enforce sampling if required
   834  			e.TraceSampled = e.TraceSampled || traceSampled
   835  			return true
   836  		}
   837  	}
   838  	return false
   839  }
   840  
   841  // As per format described at https://www.w3.org/TR/trace-context/#traceparent-header-field-values
   842  var validTraceParentExpression = regexp.MustCompile(`^(00)-([a-fA-F\d]{32})-([a-f\d]{16})-([a-fA-F\d]{2})$`)
   843  
   844  func deconstructTraceParent(s string) (traceID, spanID string, traceSampled bool) {
   845  	matches := validTraceParentExpression.FindStringSubmatch(s)
   846  	if matches != nil {
   847  		// regexp package does not support negative lookahead preventing all 0 validations
   848  		if matches[2] == "00000000000000000000000000000000" || matches[3] == "0000000000000000" {
   849  			return
   850  		}
   851  		flags, err := strconv.ParseInt(matches[4], 16, 16)
   852  		if err == nil {
   853  			traceSampled = (flags & 0x01) == 1
   854  		}
   855  		traceID, spanID = matches[2], matches[3]
   856  	}
   857  	return
   858  }
   859  
   860  var validXCloudTraceContext = regexp.MustCompile(
   861  	// Matches on "TRACE_ID"
   862  	`([a-f\d]+)?` +
   863  		// Matches on "/SPAN_ID"
   864  		`(?:/([a-f\d]+))?` +
   865  		// Matches on ";0=TRACE_TRUE"
   866  		`(?:;o=(\d))?`)
   867  
   868  func deconstructXCloudTraceContext(s string) (traceID, spanID string, traceSampled bool) {
   869  	// As per the format described at https://cloud.google.com/trace/docs/setup#force-trace
   870  	//    "X-Cloud-Trace-Context: TRACE_ID/SPAN_ID;o=TRACE_TRUE"
   871  	// for example:
   872  	//    "X-Cloud-Trace-Context: 105445aa7843bc8bf206b120001000/1;o=1"
   873  	//
   874  	// We expect:
   875  	//   * traceID (optional): 			"105445aa7843bc8bf206b120001000"
   876  	//   * spanID (optional):       	"1"
   877  	//   * traceSampled (optional): 	true
   878  	matches := validXCloudTraceContext.FindStringSubmatch(s)
   879  
   880  	if matches != nil {
   881  		traceID, spanID, traceSampled = matches[1], matches[2], matches[3] == "1"
   882  	}
   883  
   884  	if spanID == "0" {
   885  		spanID = ""
   886  	}
   887  
   888  	return
   889  }
   890  
   891  // ToLogEntry takes an Entry structure and converts it to the LogEntry proto.
   892  // A parent can take any of the following forms:
   893  //
   894  //	projects/PROJECT_ID
   895  //	folders/FOLDER_ID
   896  //	billingAccounts/ACCOUNT_ID
   897  //	organizations/ORG_ID
   898  //
   899  // for backwards compatibility, a string with no '/' is also allowed and is interpreted
   900  // as a project ID.
   901  //
   902  // ToLogEntry is implied when users invoke Logger.Log or Logger.LogSync,
   903  // but its exported as a pub function here to give users additional flexibility
   904  // when using the library. Don't call this method manually if Logger.Log or
   905  // Logger.LogSync are used, it is intended to be used together with direct call
   906  // to WriteLogEntries method.
   907  func ToLogEntry(e Entry, parent string) (*logpb.LogEntry, error) {
   908  	var l Logger
   909  	return l.ToLogEntry(e, parent)
   910  }
   911  
   912  // ToLogEntry for Logger instance
   913  func (l *Logger) ToLogEntry(e Entry, parent string) (*logpb.LogEntry, error) {
   914  	parent, err := makeParent(parent)
   915  	if err != nil {
   916  		return nil, err
   917  	}
   918  	return toLogEntryInternal(e, l, parent, 1)
   919  }
   920  
   921  func toLogEntryInternalImpl(e Entry, l *Logger, parent string, skipLevels int) (*logpb.LogEntry, error) {
   922  	if e.LogName != "" {
   923  		return nil, errors.New("logging: Entry.LogName should be not be set when writing")
   924  	}
   925  	t := e.Timestamp
   926  	if t.IsZero() {
   927  		t = now()
   928  	}
   929  	ts := timestamppb.New(t)
   930  	if l != nil && l.populateSourceLocation != DoNotPopulateSourceLocation && e.SourceLocation == nil {
   931  		if l.populateSourceLocation == AlwaysPopulateSourceLocation ||
   932  			l.populateSourceLocation == PopulateSourceLocationForDebugEntries && e.Severity == Severity(Debug) {
   933  			// filename and line are captured for source code that calls
   934  			// skipLevels up the goroutine calling stack + 1 for this func.
   935  			pc, file, line, ok := runtime.Caller(skipLevels + 1)
   936  			if ok {
   937  				details := runtime.FuncForPC(pc)
   938  				e.SourceLocation = &logpb.LogEntrySourceLocation{
   939  					File:     file,
   940  					Function: details.Name(),
   941  					Line:     int64(line),
   942  				}
   943  			}
   944  		}
   945  	}
   946  	if e.Trace == "" {
   947  		populateTraceInfo(&e, nil)
   948  		// format trace
   949  		if e.Trace != "" && !strings.Contains(e.Trace, "/traces/") {
   950  			e.Trace = fmt.Sprintf("%s/traces/%s", parent, e.Trace)
   951  		}
   952  	}
   953  	req, err := fromHTTPRequest(e.HTTPRequest)
   954  	if err != nil {
   955  		if l != nil && l.client != nil {
   956  			l.client.error(err)
   957  		} else {
   958  			return nil, err
   959  		}
   960  	}
   961  	ent := &logpb.LogEntry{
   962  		Timestamp:      ts,
   963  		Severity:       logtypepb.LogSeverity(e.Severity),
   964  		InsertId:       e.InsertID,
   965  		HttpRequest:    req,
   966  		Operation:      e.Operation,
   967  		Labels:         e.Labels,
   968  		Trace:          e.Trace,
   969  		SpanId:         e.SpanID,
   970  		Resource:       e.Resource,
   971  		SourceLocation: e.SourceLocation,
   972  		TraceSampled:   e.TraceSampled,
   973  	}
   974  	switch p := e.Payload.(type) {
   975  	case string:
   976  		ent.Payload = &logpb.LogEntry_TextPayload{TextPayload: p}
   977  	case *anypb.Any:
   978  		ent.Payload = &logpb.LogEntry_ProtoPayload{ProtoPayload: p}
   979  	default:
   980  		s, err := toProtoStruct(p)
   981  		if err != nil {
   982  			return nil, err
   983  		}
   984  		ent.Payload = &logpb.LogEntry_JsonPayload{JsonPayload: s}
   985  	}
   986  	return ent, nil
   987  }
   988  
   989  // entry represents the fields of a logging.Entry that can be parsed by Logging agent.
   990  // See the mappings at https://cloud.google.com/logging/docs/structured-logging#special-payload-fields
   991  type structuredLogEntry struct {
   992  	Message        json.RawMessage                   `json:"message"`
   993  	Severity       string                            `json:"severity,omitempty"`
   994  	HTTPRequest    *structuredLogEntryHTTPRequest    `json:"httpRequest,omitempty"`
   995  	Timestamp      string                            `json:"timestamp,omitempty"`
   996  	Labels         map[string]string                 `json:"logging.googleapis.com/labels,omitempty"`
   997  	InsertID       string                            `json:"logging.googleapis.com/insertId,omitempty"`
   998  	Operation      *structuredLogEntryOperation      `json:"logging.googleapis.com/operation,omitempty"`
   999  	SourceLocation *structuredLogEntrySourceLocation `json:"logging.googleapis.com/sourceLocation,omitempty"`
  1000  	SpanID         string                            `json:"logging.googleapis.com/spanId,omitempty"`
  1001  	Trace          string                            `json:"logging.googleapis.com/trace,omitempty"`
  1002  	TraceSampled   bool                              `json:"logging.googleapis.com/trace_sampled,omitempty"`
  1003  }
  1004  
  1005  // structuredLogEntryHTTPRequest wraps the HTTPRequest proto field in structuredLogEntry for easier JSON marshalling.
  1006  type structuredLogEntryHTTPRequest struct {
  1007  	request *logtypepb.HttpRequest
  1008  }
  1009  
  1010  func (s structuredLogEntryHTTPRequest) MarshalJSON() ([]byte, error) {
  1011  	return protojson.Marshal(s.request)
  1012  }
  1013  
  1014  // structuredLogEntryOperation wraps the Operation proto field in structuredLogEntry for easier JSON marshalling.
  1015  type structuredLogEntryOperation struct {
  1016  	operation *logpb.LogEntryOperation
  1017  }
  1018  
  1019  func (s structuredLogEntryOperation) MarshalJSON() ([]byte, error) {
  1020  	return protojson.Marshal(s.operation)
  1021  }
  1022  
  1023  // structuredLogEntrySourceLocation wraps the SourceLocation proto field in structuredLogEntry for easier JSON marshalling.
  1024  type structuredLogEntrySourceLocation struct {
  1025  	sourceLocation *logpb.LogEntrySourceLocation
  1026  }
  1027  
  1028  func (s structuredLogEntrySourceLocation) MarshalJSON() ([]byte, error) {
  1029  	return protojson.Marshal(s.sourceLocation)
  1030  }
  1031  
  1032  func serializeEntryToWriter(entry *logpb.LogEntry, w io.Writer) error {
  1033  	var httpRequest *structuredLogEntryHTTPRequest
  1034  	if entry.HttpRequest != nil {
  1035  		httpRequest = &structuredLogEntryHTTPRequest{entry.HttpRequest}
  1036  	}
  1037  
  1038  	var operation *structuredLogEntryOperation
  1039  	if entry.Operation != nil {
  1040  		operation = &structuredLogEntryOperation{entry.Operation}
  1041  	}
  1042  
  1043  	var sourceLocation *structuredLogEntrySourceLocation
  1044  	if entry.SourceLocation != nil {
  1045  		sourceLocation = &structuredLogEntrySourceLocation{entry.SourceLocation}
  1046  	}
  1047  
  1048  	jsonifiedEntry := structuredLogEntry{
  1049  		Severity:       entry.Severity.String(),
  1050  		HTTPRequest:    httpRequest,
  1051  		Timestamp:      entry.Timestamp.String(),
  1052  		Labels:         entry.Labels,
  1053  		InsertID:       entry.InsertId,
  1054  		Operation:      operation,
  1055  		SourceLocation: sourceLocation,
  1056  		SpanID:         entry.SpanId,
  1057  		Trace:          entry.Trace,
  1058  		TraceSampled:   entry.TraceSampled,
  1059  	}
  1060  	var err error
  1061  	if entry.GetTextPayload() != "" {
  1062  		jsonifiedEntry.Message, err = json.Marshal(entry.GetTextPayload())
  1063  	} else if entry.GetJsonPayload() != nil {
  1064  		jsonifiedEntry.Message, err = json.Marshal(entry.GetJsonPayload().AsMap())
  1065  	} else {
  1066  		return ErrRedirectProtoPayloadNotSupported
  1067  	}
  1068  	if err == nil {
  1069  		err = json.NewEncoder(w).Encode(jsonifiedEntry)
  1070  	}
  1071  	return err
  1072  }
  1073  

View as plain text