bench.go

Documentation: github.com/tetratelabs/wazero/internal/integration_test/vs

     1  package vs
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"os"
     7  	"path"
     8  	"runtime"
     9  	"sort"
    10  	"testing"
    11  	"text/tabwriter"
    12  
    13  	"github.com/tetratelabs/wazero/internal/testing/require"
    14  )
    15  
    16  // testCtx is an arbitrary, non-default context. Non-nil also prevents linter errors.
    17  var testCtx = context.WithValue(context.Background(), struct{}{}, "arbitrary")
    18  
    19  // ensureCompilerFastest is overridable via ldflags. e.g.
    20  //
    21  //	-ldflags '-X github.com/tetratelabs/wazero/internal/integration_test/vs.ensureCompilerFastest=true'
    22  var ensureCompilerFastest = "false"
    23  
    24  const compilerRuntime = "wazero-compiler"
    25  
    26  // runTestBenchmark_Call_CompilerFastest ensures that Compiler is the fastest engine for function invocations.
    27  // This is disabled by default, and can be run with -ldflags '-X github.com/tetratelabs/wazero/vs.ensureCompilerFastest=true'.
    28  func runTestBenchmark_Call_CompilerFastest(t *testing.T, rtCfg *RuntimeConfig, name string, call func(Module, int) error, vsRuntime Runtime) {
    29  	if ensureCompilerFastest != "true" {
    30  		t.Skip()
    31  	}
    32  
    33  	type benchResult struct {
    34  		name string
    35  		nsOp float64
    36  	}
    37  
    38  	results := make([]benchResult, 0, 2)
    39  	// Add the result for Compiler
    40  	compilerNsOp := runCallBenchmark(NewWazeroCompilerRuntime(), rtCfg, call)
    41  	results = append(results, benchResult{name: compilerRuntime, nsOp: compilerNsOp})
    42  
    43  	// Add a result for the runtime we're comparing against
    44  	vsNsOp := runCallBenchmark(vsRuntime, rtCfg, call)
    45  	results = append(results, benchResult{name: vsRuntime.Name(), nsOp: vsNsOp})
    46  
    47  	sort.Slice(results, func(i, j int) bool {
    48  		return results[i].nsOp < results[j].nsOp
    49  	})
    50  
    51  	// Print results before deciding if this failed
    52  	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
    53  	_, _ = fmt.Fprintf(w, "Benchmark%s/Call-16\n", name)
    54  	for _, result := range results {
    55  		_, _ = fmt.Fprintf(w, "%s\t%.2f\tns/op\n", result.name, result.nsOp)
    56  	}
    57  	_ = w.Flush()
    58  
    59  	// Fail if compiler wasn't fastest!
    60  	require.Equal(t, compilerRuntime, results[0].name, "%s is faster than %s. "+
    61  		"Run with ensureCompilerFastest=false instead to see the detailed result",
    62  		results[0].name, compilerRuntime)
    63  }
    64  
    65  func runCallBenchmark(rt Runtime, rtCfg *RuntimeConfig, call func(Module, int) error) float64 {
    66  	result := testing.Benchmark(func(b *testing.B) {
    67  		benchmarkCall(b, rt, rtCfg, call)
    68  	})
    69  	// https://github.com/golang/go/blob/go1.20/src/testing/benchmark.go#L428-L432
    70  	nsOp := float64(result.T.Nanoseconds()) / float64(result.N)
    71  	return nsOp
    72  }
    73  
    74  func benchmark(b *testing.B, runtime func() Runtime, rtCfg *RuntimeConfig, call func(Module, int) error) {
    75  	rt := runtime()
    76  	b.Run("Compile", func(b *testing.B) {
    77  		b.ReportAllocs()
    78  		benchmarkCompile(b, rt, rtCfg)
    79  	})
    80  	b.Run("Instantiate", func(b *testing.B) {
    81  		b.ReportAllocs()
    82  		benchmarkInstantiate(b, rt, rtCfg)
    83  	})
    84  
    85  	// Don't burn CPU when this is already going to be called in runTestBenchmark_Call_CompilerFastest
    86  	if call != nil && (ensureCompilerFastest != "true" || rt.Name() == compilerRuntime) {
    87  		b.Run("Call", func(b *testing.B) {
    88  			b.ReportAllocs()
    89  			benchmarkCall(b, rt, rtCfg, call)
    90  		})
    91  	}
    92  }
    93  
    94  func benchmarkCompile(b *testing.B, rt Runtime, rtCfg *RuntimeConfig) {
    95  	for i := 0; i < b.N; i++ {
    96  		if err := rt.Compile(testCtx, rtCfg); err != nil {
    97  			b.Fatal(err)
    98  		}
    99  		if err := rt.Close(testCtx); err != nil {
   100  			b.Fatal(err)
   101  		}
   102  	}
   103  }
   104  
   105  func benchmarkInstantiate(b *testing.B, rt Runtime, rtCfg *RuntimeConfig) {
   106  	// Compile outside the benchmark loop
   107  	if err := rt.Compile(testCtx, rtCfg); err != nil {
   108  		b.Fatal(err)
   109  	}
   110  	defer rt.Close(testCtx)
   111  
   112  	b.ResetTimer()
   113  	for i := 0; i < b.N; i++ {
   114  		mod, err := rt.Instantiate(testCtx, rtCfg)
   115  		if err != nil {
   116  			b.Fatal(err)
   117  		}
   118  		err = mod.Close(testCtx)
   119  		if err != nil {
   120  			b.Fatal(err)
   121  		}
   122  	}
   123  }
   124  
   125  func benchmarkCall(b *testing.B, rt Runtime, rtCfg *RuntimeConfig, call func(Module, int) error) {
   126  	// Initialize outside the benchmark loop
   127  	if err := rt.Compile(testCtx, rtCfg); err != nil {
   128  		b.Fatal(err)
   129  	}
   130  	defer rt.Close(testCtx)
   131  	mod, err := rt.Instantiate(testCtx, rtCfg)
   132  	if err != nil {
   133  		b.Fatal(err)
   134  	}
   135  	defer mod.Close(testCtx)
   136  	b.ResetTimer()
   137  	for i := 0; i < b.N; i++ {
   138  		if err := call(mod, i); err != nil {
   139  			b.Fatal(err)
   140  		}
   141  	}
   142  }
   143  
   144  func testCall(t *testing.T, runtime func() Runtime, rtCfg *RuntimeConfig, testCall func(*testing.T, Module, int, int)) {
   145  	rt := runtime()
   146  	err := rt.Compile(testCtx, rtCfg)
   147  	require.NoError(t, err)
   148  	defer rt.Close(testCtx)
   149  
   150  	// Ensure the module can be re-instantiated times, even if not all runtimes allow renaming.
   151  	for i := 0; i < 10; i++ {
   152  		m, err := rt.Instantiate(testCtx, rtCfg)
   153  		require.NoError(t, err)
   154  
   155  		// Large loop in test is only to show the function is stable (ex doesn't leak or crash on Nth use).
   156  		for j := 0; j < 1000; j++ {
   157  			testCall(t, m, i, j)
   158  		}
   159  
   160  		require.NoError(t, m.Close(testCtx))
   161  	}
   162  }
   163  
   164  func testInstantiate(t *testing.T, runtime func() Runtime, rtCfg *RuntimeConfig) {
   165  	rt := runtime()
   166  	err := rt.Compile(testCtx, rtCfg)
   167  	require.NoError(t, err)
   168  	defer rt.Close(testCtx)
   169  
   170  	// Ensure the module can be re-instantiated times, even if not all runtimes allow renaming.
   171  	for i := 0; i < 10; i++ {
   172  		m, err := rt.Instantiate(testCtx, rtCfg)
   173  		require.NoError(t, err)
   174  		require.NoError(t, m.Close(testCtx))
   175  	}
   176  }
   177  
   178  func readRelativeFile(relativePath string) []byte {
   179  	// We can't resolve relative paths as init() is called from each of its subdirs
   180  	_, source, _, _ := runtime.Caller(1) // 1 as this utility is in a different source than the caller.
   181  	realPath := path.Join(path.Dir(source), relativePath)
   182  	bytes, err := os.ReadFile(realPath)
   183  	if err != nil {
   184  		panic(err)
   185  	}
   186  	return bytes
   187  }
   188
View as plain text