compare.go

Documentation: golang.org/x/tools/cmd/benchcmp

     1  // Copyright 2014 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package main
     6  
     7  import (
     8  	"fmt"
     9  	"math"
    10  
    11  	"golang.org/x/tools/benchmark/parse"
    12  )
    13  
    14  // BenchCmp is a pair of benchmarks.
    15  type BenchCmp struct {
    16  	Before *parse.Benchmark
    17  	After  *parse.Benchmark
    18  }
    19  
    20  // Correlate correlates benchmarks from two BenchSets.
    21  func Correlate(before, after parse.Set) (cmps []BenchCmp, warnings []string) {
    22  	cmps = make([]BenchCmp, 0, len(after))
    23  	for name, beforebb := range before {
    24  		afterbb := after[name]
    25  		if len(beforebb) != len(afterbb) {
    26  			warnings = append(warnings, fmt.Sprintf("ignoring %s: before has %d instances, after has %d", name, len(beforebb), len(afterbb)))
    27  			continue
    28  		}
    29  		for i, beforeb := range beforebb {
    30  			afterb := afterbb[i]
    31  			cmps = append(cmps, BenchCmp{beforeb, afterb})
    32  		}
    33  	}
    34  	return
    35  }
    36  
    37  func (c BenchCmp) Name() string           { return c.Before.Name }
    38  func (c BenchCmp) String() string         { return fmt.Sprintf("<%s, %s>", c.Before, c.After) }
    39  func (c BenchCmp) Measured(flag int) bool { return (c.Before.Measured & c.After.Measured & flag) != 0 }
    40  func (c BenchCmp) DeltaNsPerOp() Delta    { return Delta{c.Before.NsPerOp, c.After.NsPerOp} }
    41  func (c BenchCmp) DeltaMBPerS() Delta     { return Delta{c.Before.MBPerS, c.After.MBPerS} }
    42  func (c BenchCmp) DeltaAllocedBytesPerOp() Delta {
    43  	return Delta{float64(c.Before.AllocedBytesPerOp), float64(c.After.AllocedBytesPerOp)}
    44  }
    45  func (c BenchCmp) DeltaAllocsPerOp() Delta {
    46  	return Delta{float64(c.Before.AllocsPerOp), float64(c.After.AllocsPerOp)}
    47  }
    48  
    49  // Delta is the before and after value for a benchmark measurement.
    50  // Both must be non-negative.
    51  type Delta struct {
    52  	Before float64
    53  	After  float64
    54  }
    55  
    56  // mag calculates the magnitude of a change, regardless of the direction of
    57  // the change. mag is intended for sorting and has no independent meaning.
    58  func (d Delta) mag() float64 {
    59  	switch {
    60  	case d.Before != 0 && d.After != 0 && d.Before >= d.After:
    61  		return d.After / d.Before
    62  	case d.Before != 0 && d.After != 0 && d.Before < d.After:
    63  		return d.Before / d.After
    64  	case d.Before == 0 && d.After == 0:
    65  		return 1
    66  	default:
    67  		// 0 -> 1 or 1 -> 0
    68  		// These are significant changes and worth surfacing.
    69  		return math.Inf(1)
    70  	}
    71  }
    72  
    73  // Changed reports whether the benchmark quantities are different.
    74  func (d Delta) Changed() bool { return d.Before != d.After }
    75  
    76  // Float64 returns After / Before. If Before is 0, Float64 returns
    77  // 1 if After is also 0, and +Inf otherwise.
    78  func (d Delta) Float64() float64 {
    79  	switch {
    80  	case d.Before != 0:
    81  		return d.After / d.Before
    82  	case d.After == 0:
    83  		return 1
    84  	default:
    85  		return math.Inf(1)
    86  	}
    87  }
    88  
    89  // Percent formats a Delta as a percent change, ranging from -100% up.
    90  func (d Delta) Percent() string {
    91  	return fmt.Sprintf("%+.2f%%", 100*d.Float64()-100)
    92  }
    93  
    94  // Multiple formats a Delta as a multiplier, ranging from 0.00x up.
    95  func (d Delta) Multiple() string {
    96  	return fmt.Sprintf("%.2fx", d.Float64())
    97  }
    98  
    99  func (d Delta) String() string {
   100  	return fmt.Sprintf("Δ(%f, %f)", d.Before, d.After)
   101  }
   102  
   103  // ByParseOrder sorts BenchCmps to match the order in
   104  // which the Before benchmarks were presented to Parse.
   105  type ByParseOrder []BenchCmp
   106  
   107  func (x ByParseOrder) Len() int           { return len(x) }
   108  func (x ByParseOrder) Swap(i, j int)      { x[i], x[j] = x[j], x[i] }
   109  func (x ByParseOrder) Less(i, j int) bool { return x[i].Before.Ord < x[j].Before.Ord }
   110  
   111  // lessByDelta provides lexicographic ordering:
   112  //   - largest delta by magnitude
   113  //   - alphabetic by name
   114  func lessByDelta(i, j BenchCmp, calcDelta func(BenchCmp) Delta) bool {
   115  	iDelta, jDelta := calcDelta(i).mag(), calcDelta(j).mag()
   116  	if iDelta != jDelta {
   117  		return iDelta < jDelta
   118  	}
   119  	return i.Name() < j.Name()
   120  }
   121  
   122  // ByDeltaNsPerOp sorts BenchCmps lexicographically by change
   123  // in ns/op, descending, then by benchmark name.
   124  type ByDeltaNsPerOp []BenchCmp
   125  
   126  func (x ByDeltaNsPerOp) Len() int           { return len(x) }
   127  func (x ByDeltaNsPerOp) Swap(i, j int)      { x[i], x[j] = x[j], x[i] }
   128  func (x ByDeltaNsPerOp) Less(i, j int) bool { return lessByDelta(x[i], x[j], BenchCmp.DeltaNsPerOp) }
   129  
   130  // ByDeltaMBPerS sorts BenchCmps lexicographically by change
   131  // in MB/s, descending, then by benchmark name.
   132  type ByDeltaMBPerS []BenchCmp
   133  
   134  func (x ByDeltaMBPerS) Len() int           { return len(x) }
   135  func (x ByDeltaMBPerS) Swap(i, j int)      { x[i], x[j] = x[j], x[i] }
   136  func (x ByDeltaMBPerS) Less(i, j int) bool { return lessByDelta(x[i], x[j], BenchCmp.DeltaMBPerS) }
   137  
   138  // ByDeltaAllocedBytesPerOp sorts BenchCmps lexicographically by change
   139  // in B/op, descending, then by benchmark name.
   140  type ByDeltaAllocedBytesPerOp []BenchCmp
   141  
   142  func (x ByDeltaAllocedBytesPerOp) Len() int      { return len(x) }
   143  func (x ByDeltaAllocedBytesPerOp) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
   144  func (x ByDeltaAllocedBytesPerOp) Less(i, j int) bool {
   145  	return lessByDelta(x[i], x[j], BenchCmp.DeltaAllocedBytesPerOp)
   146  }
   147  
   148  // ByDeltaAllocsPerOp sorts BenchCmps lexicographically by change
   149  // in allocs/op, descending, then by benchmark name.
   150  type ByDeltaAllocsPerOp []BenchCmp
   151  
   152  func (x ByDeltaAllocsPerOp) Len() int      { return len(x) }
   153  func (x ByDeltaAllocsPerOp) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
   154  func (x ByDeltaAllocsPerOp) Less(i, j int) bool {
   155  	return lessByDelta(x[i], x[j], BenchCmp.DeltaAllocsPerOp)
   156  }
   157
View as plain text