middleware.go

Documentation: github.com/ory/x/metricsx

     1  /*
     2   * Copyright © 2015-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   *
    16   * @author		Aeneas Rekkas <aeneas+oss@aeneas.io>
    17   * @copyright 	2015-2018 Aeneas Rekkas <aeneas+oss@aeneas.io>
    18   * @license 	Apache-2.0
    19   */
    20  
    21  package metricsx
    22  
    23  import (
    24  	"crypto/sha256"
    25  	"encoding/hex"
    26  	"fmt"
    27  	"net"
    28  	"net/http"
    29  	"net/url"
    30  	"runtime"
    31  	"strings"
    32  	"sync"
    33  	"time"
    34  
    35  	"github.com/ory/x/configx"
    36  
    37  	"github.com/spf13/cobra"
    38  
    39  	"github.com/ory/x/cmdx"
    40  	"github.com/ory/x/logrusx"
    41  	"github.com/ory/x/resilience"
    42  
    43  	"github.com/pborman/uuid"
    44  	"github.com/urfave/negroni"
    45  
    46  	analytics "github.com/ory/analytics-go/v4"
    47  )
    48  
    49  var instance *Service
    50  var lock sync.Mutex
    51  
    52  // Service helps with providing context on metrics.
    53  type Service struct {
    54  	optOut bool
    55  	salt   string
    56  
    57  	o       *Options
    58  	context *analytics.Context
    59  
    60  	c analytics.Client
    61  	l *logrusx.Logger
    62  
    63  	mem *MemoryStatistics
    64  }
    65  
    66  // Hash returns a hashed string of the value.
    67  func Hash(value string) string {
    68  	hash := sha256.New()
    69  	_, err := hash.Write([]byte(value))
    70  	if err != nil {
    71  		panic(fmt.Sprintf("unable to hash value"))
    72  	}
    73  	return hex.EncodeToString(hash.Sum(nil))
    74  }
    75  
    76  // Options configures the metrics service.
    77  type Options struct {
    78  	// Service represents the service name, for example "ory-hydra".
    79  	Service string
    80  
    81  	// ClusterID represents the cluster id, typically a hash of some unique configuration properties.
    82  	ClusterID string
    83  
    84  	// IsDevelopment should be true if we assume that we're in a development environment.
    85  	IsDevelopment bool
    86  
    87  	// WriteKey is the segment API key.
    88  	WriteKey string
    89  
    90  	// WhitelistedPaths represents a list of paths that can be transmitted in clear text to segment.
    91  	WhitelistedPaths []string
    92  
    93  	// BuildVersion represents the build version.
    94  	BuildVersion string
    95  
    96  	// BuildHash represents the build git hash.
    97  	BuildHash string
    98  
    99  	// BuildTime represents the build time.
   100  	BuildTime string
   101  
   102  	// Config overrides the analytics.Config. If nil, sensible defaults will be used.
   103  	Config *analytics.Config
   104  
   105  	// MemoryInterval sets how often memory statistics should be transmitted. Defaults to every 12 hours.
   106  	MemoryInterval time.Duration
   107  }
   108  
   109  type void struct {
   110  }
   111  
   112  func (v *void) Logf(format string, args ...interface{}) {
   113  }
   114  
   115  func (v *void) Errorf(format string, args ...interface{}) {
   116  }
   117  
   118  // New returns a new metrics service. If one has been instantiated already, no new instance will be created.
   119  func New(
   120  	cmd *cobra.Command,
   121  	l *logrusx.Logger,
   122  	c *configx.Provider,
   123  	o *Options,
   124  ) *Service {
   125  	lock.Lock()
   126  	defer lock.Unlock()
   127  
   128  	if instance != nil {
   129  		return instance
   130  	}
   131  
   132  	if o.BuildTime == "" {
   133  		o.BuildTime = "unknown"
   134  	}
   135  
   136  	if o.BuildVersion == "" {
   137  		o.BuildVersion = "unknown"
   138  	}
   139  
   140  	if o.BuildHash == "" {
   141  		o.BuildHash = "unknown"
   142  	}
   143  
   144  	if o.Config == nil {
   145  		o.Config = &analytics.Config{
   146  			Interval: time.Hour * 24,
   147  		}
   148  	}
   149  
   150  	o.Config.Logger = new(void)
   151  
   152  	if o.MemoryInterval < time.Minute {
   153  		o.MemoryInterval = time.Hour * 12
   154  	}
   155  
   156  	segment, err := analytics.NewWithConfig(o.WriteKey, *o.Config)
   157  	if err != nil {
   158  		l.WithError(err).Fatalf("Unable to initialise software quality assurance features.")
   159  		return nil
   160  	}
   161  
   162  	var oi analytics.OSInfo
   163  
   164  	optOut, err := cmd.Flags().GetBool("sqa-opt-out")
   165  	if err != nil {
   166  		cmdx.Must(err, `Unable to get command line flag "sqa-opt-out": %s`, err)
   167  	}
   168  
   169  	if !optOut {
   170  		optOut = c.Bool("sqa.opt_out")
   171  	}
   172  
   173  	if !optOut {
   174  		l.Info("Software quality assurance features are enabled. Learn more at: https://www.ory.sh/docs/ecosystem/sqa")
   175  		oi = analytics.OSInfo{
   176  			Version: fmt.Sprintf("%s-%s", runtime.GOOS, runtime.GOARCH),
   177  		}
   178  	}
   179  
   180  	m := &Service{
   181  		optOut: optOut,
   182  		salt:   uuid.New(),
   183  		o:      o,
   184  		c:      segment,
   185  		l:      l,
   186  		mem:    new(MemoryStatistics),
   187  		context: &analytics.Context{
   188  			IP: net.IPv4(0, 0, 0, 0),
   189  			App: analytics.AppInfo{
   190  				Name:    o.Service,
   191  				Version: o.BuildVersion,
   192  				Build:   fmt.Sprintf("%s/%s/%s", o.BuildVersion, o.BuildHash, o.BuildTime),
   193  			},
   194  			OS: oi,
   195  			Traits: analytics.NewTraits().
   196  				Set("optedOut", optOut).
   197  				Set("instanceId", uuid.New()).
   198  				Set("isDevelopment", o.IsDevelopment),
   199  			UserAgent: "github.com/ory/x/metricsx.Service/v0.0.1",
   200  		},
   201  	}
   202  
   203  	instance = m
   204  
   205  	go m.Identify()
   206  	go m.ObserveMemory()
   207  
   208  	return m
   209  }
   210  
   211  // Identify enables reporting to segment.
   212  func (sw *Service) Identify() {
   213  	if err := resilience.Retry(sw.l, time.Minute*5, time.Hour*24*30, func() error {
   214  		return sw.c.Enqueue(analytics.Identify{
   215  			UserId:  sw.o.ClusterID,
   216  			Traits:  sw.context.Traits,
   217  			Context: sw.context,
   218  		})
   219  	}); err != nil {
   220  		sw.l.WithError(err).Debug("Could not commit anonymized environment information")
   221  	}
   222  }
   223  
   224  // ObserveMemory commits memory statistics to segment.
   225  func (sw *Service) ObserveMemory() {
   226  	if sw.optOut {
   227  		return
   228  	}
   229  
   230  	for {
   231  		sw.mem.Update()
   232  		if err := sw.c.Enqueue(analytics.Track{
   233  			UserId:     sw.o.ClusterID,
   234  			Event:      "memstats",
   235  			Properties: analytics.Properties(sw.mem.ToMap()),
   236  			Context:    sw.context,
   237  		}); err != nil {
   238  			sw.l.WithError(err).Debug("Could not commit anonymized telemetry data")
   239  		}
   240  		time.Sleep(sw.o.MemoryInterval)
   241  	}
   242  }
   243  
   244  // ServeHTTP is a middleware for sending meta information to segment.
   245  func (sw *Service) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
   246  	var start time.Time
   247  	if !sw.optOut {
   248  		start = time.Now()
   249  	}
   250  
   251  	next(rw, r)
   252  
   253  	if sw.optOut {
   254  		return
   255  	}
   256  
   257  	latency := time.Now().UTC().Sub(start.UTC()) / time.Millisecond
   258  
   259  	scheme := "https:"
   260  	if r.TLS == nil {
   261  		scheme = "http:"
   262  	}
   263  
   264  	path := sw.anonymizePath(r.URL.Path, sw.salt)
   265  	query := sw.anonymizeQuery(r.URL.Query(), sw.salt)
   266  
   267  	// Collecting request info
   268  	res := rw.(negroni.ResponseWriter)
   269  	status := res.Status()
   270  	size := res.Size()
   271  
   272  	if err := sw.c.Enqueue(analytics.Page{
   273  		UserId: sw.o.ClusterID,
   274  		Name:   path,
   275  		Properties: analytics.
   276  			NewProperties().
   277  			SetURL(scheme+"//"+sw.o.ClusterID+path+"?"+query).
   278  			SetPath(path).
   279  			SetName(path).
   280  			Set("status", status).
   281  			Set("size", size).
   282  			Set("latency", latency).
   283  			Set("method", r.Method),
   284  		Context: sw.context,
   285  	}); err != nil {
   286  		sw.l.WithError(err).Debug("Could not commit anonymized telemetry data")
   287  		// do nothing...
   288  	}
   289  }
   290  
   291  func (sw *Service) Close() error {
   292  	return sw.c.Close()
   293  }
   294  
   295  func (sw *Service) anonymizePath(path string, salt string) string {
   296  	paths := sw.o.WhitelistedPaths
   297  	path = strings.ToLower(path)
   298  
   299  	for _, p := range paths {
   300  		p = strings.ToLower(p)
   301  		if len(path) == len(p) && path[:len(p)] == strings.ToLower(p) {
   302  			return p
   303  		} else if len(path) > len(p) && path[:len(p)+1] == strings.ToLower(p)+"/" {
   304  			return path[:len(p)] + "/" + Hash(path[len(p):]+"|"+salt)
   305  		}
   306  	}
   307  
   308  	return "/"
   309  }
   310  
   311  func (sw *Service) anonymizeQuery(query url.Values, salt string) string {
   312  	for _, q := range query {
   313  		for i, s := range q {
   314  			if s != "" {
   315  				s = Hash(s + "|" + salt)
   316  				q[i] = s
   317  			}
   318  		}
   319  	}
   320  	return query.Encode()
   321  }
   322
View as plain text