bench_test.go

Documentation: github.com/tetratelabs/wazero/internal/integration_test/bench

     1  package bench
     2  
     3  import (
     4  	"context"
     5  	"crypto/rand"
     6  	_ "embed"
     7  	"fmt"
     8  	"runtime"
     9  	"testing"
    10  
    11  	"github.com/tetratelabs/wazero"
    12  	"github.com/tetratelabs/wazero/api"
    13  	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
    14  	"github.com/tetratelabs/wazero/internal/platform"
    15  )
    16  
    17  // testCtx is an arbitrary, non-default context. Non-nil also prevents linter errors.
    18  var testCtx = context.WithValue(context.Background(), struct{}{}, "arbitrary")
    19  
    20  // caseWasm was compiled from TinyGo testdata/case.go
    21  //
    22  //go:embed testdata/case.wasm
    23  var caseWasm []byte
    24  
    25  func BenchmarkInvocation(b *testing.B) {
    26  	b.Run("interpreter", func(b *testing.B) {
    27  		m := instantiateHostFunctionModuleWithEngine(b, wazero.NewRuntimeConfigInterpreter())
    28  		defer m.Close(testCtx)
    29  		runAllInvocationBenches(b, m)
    30  	})
    31  	if runtime.GOARCH == "amd64" || runtime.GOARCH == "arm64" {
    32  		b.Run("compiler", func(b *testing.B) {
    33  			m := instantiateHostFunctionModuleWithEngine(b, wazero.NewRuntimeConfigCompiler())
    34  			defer m.Close(testCtx)
    35  			runAllInvocationBenches(b, m)
    36  		})
    37  	}
    38  }
    39  
    40  func BenchmarkInitialization(b *testing.B) {
    41  	b.Run("interpreter", func(b *testing.B) {
    42  		r := createRuntime(b, wazero.NewRuntimeConfigInterpreter())
    43  		runInitializationBench(b, r)
    44  	})
    45  
    46  	b.Run("interpreter-multiple", func(b *testing.B) {
    47  		r := createRuntime(b, wazero.NewRuntimeConfigInterpreter())
    48  		runInitializationConcurrentBench(b, r)
    49  	})
    50  
    51  	if platform.CompilerSupported() {
    52  		b.Run("compiler", func(b *testing.B) {
    53  			r := createRuntime(b, wazero.NewRuntimeConfigCompiler())
    54  			runInitializationBench(b, r)
    55  		})
    56  
    57  		b.Run("compiler-multiple", func(b *testing.B) {
    58  			r := createRuntime(b, wazero.NewRuntimeConfigCompiler())
    59  			runInitializationConcurrentBench(b, r)
    60  		})
    61  	}
    62  }
    63  
    64  func BenchmarkCompilation(b *testing.B) {
    65  	if !platform.CompilerSupported() {
    66  		b.Skip()
    67  	}
    68  
    69  	// Note: recreate runtime each time in the loop to ensure that
    70  	// recompilation happens if the extern cache is not used.
    71  	b.Run("with extern cache", func(b *testing.B) {
    72  		cache, err := wazero.NewCompilationCacheWithDir(b.TempDir())
    73  		if err != nil {
    74  			b.Fatal(err)
    75  		}
    76  		for i := 0; i < b.N; i++ {
    77  			r := wazero.NewRuntimeWithConfig(context.Background(), wazero.NewRuntimeConfigCompiler().WithCompilationCache(cache))
    78  			runCompilation(b, r)
    79  		}
    80  	})
    81  	b.Run("without extern cache", func(b *testing.B) {
    82  		b.ResetTimer()
    83  		for i := 0; i < b.N; i++ {
    84  			r := wazero.NewRuntimeWithConfig(context.Background(), wazero.NewRuntimeConfigCompiler())
    85  			runCompilation(b, r)
    86  		}
    87  	})
    88  	b.Run("interpreter", func(b *testing.B) {
    89  		b.ResetTimer()
    90  		for i := 0; i < b.N; i++ {
    91  			r := wazero.NewRuntimeWithConfig(context.Background(), wazero.NewRuntimeConfigInterpreter())
    92  			runCompilation(b, r)
    93  		}
    94  	})
    95  }
    96  
    97  func runCompilation(b *testing.B, r wazero.Runtime) wazero.CompiledModule {
    98  	compiled, err := r.CompileModule(testCtx, caseWasm)
    99  	if err != nil {
   100  		b.Fatal(err)
   101  	}
   102  	return compiled
   103  }
   104  
   105  func runInitializationBench(b *testing.B, r wazero.Runtime) {
   106  	compiled := runCompilation(b, r)
   107  	defer compiled.Close(testCtx)
   108  	// Configure with real sources to avoid performance hit initializing fake ones. These sources are not used
   109  	// in the benchmark.
   110  	config := wazero.NewModuleConfig().WithSysNanotime().WithSysWalltime().WithRandSource(rand.Reader).
   111  		// To measure the pure instantiation time without including calling _start.
   112  		WithStartFunctions()
   113  	b.ResetTimer()
   114  	for i := 0; i < b.N; i++ {
   115  		mod, err := r.InstantiateModule(testCtx, compiled, config)
   116  		if err != nil {
   117  			b.Fatal(err)
   118  		}
   119  		mod.Close(testCtx)
   120  	}
   121  }
   122  
   123  func runInitializationConcurrentBench(b *testing.B, r wazero.Runtime) {
   124  	compiled := runCompilation(b, r)
   125  	defer compiled.Close(testCtx)
   126  	// Configure with real sources to avoid performance hit initializing fake ones. These sources are not used
   127  	// in the benchmark.
   128  	config := wazero.NewModuleConfig().
   129  		WithSysNanotime().
   130  		WithSysWalltime().
   131  		WithRandSource(rand.Reader).
   132  		// To measure the pure instantiation time without including calling _start.
   133  		WithStartFunctions().
   134  		WithName("")
   135  	b.RunParallel(func(pb *testing.PB) {
   136  		for pb.Next() {
   137  			m, err := r.InstantiateModule(testCtx, compiled, config)
   138  			if err != nil {
   139  				b.Error(err)
   140  			} else {
   141  				m.Close(testCtx)
   142  			}
   143  		}
   144  	})
   145  }
   146  
   147  func runAllInvocationBenches(b *testing.B, m api.Module) {
   148  	runBase64Benches(b, m)
   149  	runFibBenches(b, m)
   150  	runStringManipulationBenches(b, m)
   151  	runReverseArrayBenches(b, m)
   152  	runRandomMatMul(b, m)
   153  }
   154  
   155  func runBase64Benches(b *testing.B, m api.Module) {
   156  	base64 := m.ExportedFunction("base64")
   157  
   158  	for _, numPerExec := range []int{5, 100, 10000} {
   159  		numPerExec := uint64(numPerExec)
   160  		b.ResetTimer()
   161  		b.Run(fmt.Sprintf("base64_%d_per_exec", numPerExec), func(b *testing.B) {
   162  			for i := 0; i < b.N; i++ {
   163  				if _, err := base64.Call(testCtx, numPerExec); err != nil {
   164  					b.Fatal(err)
   165  				}
   166  			}
   167  		})
   168  	}
   169  }
   170  
   171  func runFibBenches(b *testing.B, m api.Module) {
   172  	fibonacci := m.ExportedFunction("fibonacci")
   173  
   174  	for _, num := range []int{5, 10, 20, 30} {
   175  		num := uint64(num)
   176  		b.ResetTimer()
   177  		b.Run(fmt.Sprintf("fib_for_%d", num), func(b *testing.B) {
   178  			for i := 0; i < b.N; i++ {
   179  				if _, err := fibonacci.Call(testCtx, num); err != nil {
   180  					b.Fatal(err)
   181  				}
   182  			}
   183  		})
   184  	}
   185  }
   186  
   187  func runStringManipulationBenches(b *testing.B, m api.Module) {
   188  	stringManipulation := m.ExportedFunction("string_manipulation")
   189  
   190  	for _, initialSize := range []int{50, 100, 1000} {
   191  		initialSize := uint64(initialSize)
   192  		b.ResetTimer()
   193  		b.Run(fmt.Sprintf("string_manipulation_size_%d", initialSize), func(b *testing.B) {
   194  			for i := 0; i < b.N; i++ {
   195  				if _, err := stringManipulation.Call(testCtx, initialSize); err != nil {
   196  					b.Fatal(err)
   197  				}
   198  			}
   199  		})
   200  	}
   201  }
   202  
   203  func runReverseArrayBenches(b *testing.B, m api.Module) {
   204  	reverseArray := m.ExportedFunction("reverse_array")
   205  
   206  	for _, arraySize := range []int{500, 1000, 10000} {
   207  		arraySize := uint64(arraySize)
   208  		b.ResetTimer()
   209  		b.Run(fmt.Sprintf("reverse_array_size_%d", arraySize), func(b *testing.B) {
   210  			for i := 0; i < b.N; i++ {
   211  				if _, err := reverseArray.Call(testCtx, arraySize); err != nil {
   212  					b.Fatal(err)
   213  				}
   214  			}
   215  		})
   216  	}
   217  }
   218  
   219  func runRandomMatMul(b *testing.B, m api.Module) {
   220  	randomMatMul := m.ExportedFunction("random_mat_mul")
   221  
   222  	for _, matrixSize := range []int{5, 10, 20} {
   223  		matrixSize := uint64(matrixSize)
   224  		b.ResetTimer()
   225  		b.Run(fmt.Sprintf("random_mat_mul_size_%d", matrixSize), func(b *testing.B) {
   226  			for i := 0; i < b.N; i++ {
   227  				if _, err := randomMatMul.Call(testCtx, matrixSize); err != nil {
   228  					b.Fatal(err)
   229  				}
   230  			}
   231  		})
   232  	}
   233  }
   234  
   235  func instantiateHostFunctionModuleWithEngine(b *testing.B, config wazero.RuntimeConfig) api.Module {
   236  	r := createRuntime(b, config)
   237  
   238  	// Instantiate runs the "_start" function which is what TinyGo compiles "main" to.
   239  	m, err := r.Instantiate(testCtx, caseWasm)
   240  	if err != nil {
   241  		b.Fatal(err)
   242  	}
   243  	return m
   244  }
   245  
   246  func createRuntime(b *testing.B, config wazero.RuntimeConfig) wazero.Runtime {
   247  	getRandomString := func(ctx context.Context, m api.Module, retBufPtr uint32, retBufSize uint32) {
   248  		results, err := m.ExportedFunction("allocate_buffer").Call(ctx, 10)
   249  		if err != nil {
   250  			b.Fatal(err)
   251  		}
   252  
   253  		offset := uint32(results[0])
   254  		m.Memory().WriteUint32Le(retBufPtr, offset)
   255  		m.Memory().WriteUint32Le(retBufSize, 10)
   256  		b := make([]byte, 10)
   257  		_, _ = rand.Read(b)
   258  		m.Memory().Write(offset, b)
   259  	}
   260  
   261  	r := wazero.NewRuntimeWithConfig(testCtx, config)
   262  
   263  	_, err := r.NewHostModuleBuilder("env").
   264  		NewFunctionBuilder().WithFunc(getRandomString).Export("get_random_string").
   265  		Instantiate(testCtx)
   266  	if err != nil {
   267  		b.Fatal(err)
   268  	}
   269  
   270  	// Note: host_func.go doesn't directly use WASI, but TinyGo needs to be initialized as a WASI Command.
   271  	// Add WASI to satisfy import tests
   272  	wasi_snapshot_preview1.MustInstantiate(testCtx, r)
   273  	return r
   274  }
   275
View as plain text