bench_test.go

Documentation: github.com/tetratelabs/wazero/internal/integration_test/stdlibs

     1  package wazevo_test
     2  
     3  import (
     4  	"context"
     5  	"crypto/rand"
     6  	"io"
     7  	"os"
     8  	"path/filepath"
     9  	"runtime"
    10  	"strings"
    11  	"testing"
    12  
    13  	"github.com/tetratelabs/wazero"
    14  	"github.com/tetratelabs/wazero/experimental/opt"
    15  	"github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1"
    16  	"github.com/tetratelabs/wazero/internal/testing/require"
    17  	"github.com/tetratelabs/wazero/sys"
    18  )
    19  
    20  func BenchmarkZig(b *testing.B) {
    21  	if runtime.GOARCH == "arm64" {
    22  		b.Run("optimizing", func(b *testing.B) {
    23  			c := opt.NewRuntimeConfigOptimizingCompiler()
    24  			runtBenches(b, context.Background(), c, zigTestCase)
    25  		})
    26  	}
    27  	b.Run("baseline", func(b *testing.B) {
    28  		c := wazero.NewRuntimeConfigCompiler()
    29  		runtBenches(b, context.Background(), c, zigTestCase)
    30  	})
    31  }
    32  
    33  func BenchmarkTinyGo(b *testing.B) {
    34  	if runtime.GOARCH == "arm64" {
    35  		b.Run("optimizing", func(b *testing.B) {
    36  			c := opt.NewRuntimeConfigOptimizingCompiler()
    37  			runtBenches(b, context.Background(), c, tinyGoTestCase)
    38  		})
    39  	}
    40  	b.Run("baseline", func(b *testing.B) {
    41  		c := wazero.NewRuntimeConfigCompiler()
    42  		runtBenches(b, context.Background(), c, tinyGoTestCase)
    43  	})
    44  }
    45  
    46  func BenchmarkWasip1(b *testing.B) {
    47  	if runtime.GOARCH == "arm64" {
    48  		b.Run("optimizing", func(b *testing.B) {
    49  			c := opt.NewRuntimeConfigOptimizingCompiler()
    50  			runtBenches(b, context.Background(), c, wasip1TestCase)
    51  		})
    52  	}
    53  	b.Run("baseline", func(b *testing.B) {
    54  		c := wazero.NewRuntimeConfigCompiler()
    55  		runtBenches(b, context.Background(), c, wasip1TestCase)
    56  	})
    57  }
    58  
    59  type testCase struct {
    60  	name, dir    string
    61  	readTestCase func(fpath string, fname string) (_ []byte, c wazero.ModuleConfig, stdout, stderr *os.File, err error)
    62  }
    63  
    64  var (
    65  	zigTestCase = testCase{
    66  		name: "zig",
    67  		dir:  "testdata/zig/",
    68  		readTestCase: func(fpath string, fname string) (_ []byte, c wazero.ModuleConfig, stdout, stderr *os.File, err error) {
    69  			bin, err := os.ReadFile(fpath)
    70  			c, stdout, stderr = defaultModuleConfig()
    71  			c = c.WithFSConfig(wazero.NewFSConfig().WithDirMount(".", "/")).
    72  				WithArgs("test.wasm")
    73  			return bin, c, stdout, stderr, err
    74  		},
    75  	}
    76  	tinyGoTestCase = testCase{
    77  		name: "tinygo",
    78  		dir:  "testdata/tinygo/",
    79  		readTestCase: func(fpath string, fname string) (_ []byte, c wazero.ModuleConfig, stdout, stderr *os.File, err error) {
    80  			if !strings.HasSuffix(fname, ".test") {
    81  				return nil, nil, nil, nil, nil
    82  			}
    83  			bin, err := os.ReadFile(fpath)
    84  
    85  			fsconfig := wazero.NewFSConfig().
    86  				WithDirMount(".", "/").
    87  				WithDirMount(os.TempDir(), "/tmp")
    88  
    89  			c, stdout, stderr = defaultModuleConfig()
    90  			c = c.WithFSConfig(fsconfig).
    91  				WithArgs(fname, "-test.v")
    92  
    93  			return bin, c, stdout, stderr, err
    94  		},
    95  	}
    96  	wasip1TestCase = testCase{
    97  		name: "wasip1",
    98  		dir:  "testdata/go/",
    99  		readTestCase: func(fpath string, fname string) (_ []byte, c wazero.ModuleConfig, stdout, stderr *os.File, err error) {
   100  			if !strings.HasSuffix(fname, ".test") {
   101  				return nil, nil, nil, nil, nil
   102  			}
   103  			bin, err := os.ReadFile(fpath)
   104  			if err != nil {
   105  				return nil, nil, nil, nil, err
   106  			}
   107  			fsuffixstripped := strings.ReplaceAll(fname, ".test", "")
   108  			inferredpath := strings.ReplaceAll(fsuffixstripped, "_", "/")
   109  			testdir := filepath.Join(runtime.GOROOT(), inferredpath)
   110  			err = os.Chdir(testdir)
   111  
   112  			sysroot := filepath.VolumeName(testdir) + string(os.PathSeparator)
   113  			normalizedTestdir := normalizeOsPath(testdir)
   114  
   115  			c, stdout, stderr = defaultModuleConfig()
   116  			c = c.WithFSConfig(
   117  				wazero.NewFSConfig().
   118  					WithDirMount(sysroot, "/").
   119  					WithDirMount(os.TempDir(), "/tmp")).
   120  				WithEnv("PWD", normalizedTestdir)
   121  
   122  			args := []string{fname, "-test.short", "-test.v"}
   123  
   124  			// Skip tests that are fragile on Windows.
   125  			if runtime.GOOS == "windows" {
   126  				c = c.
   127  					WithEnv("GOROOT", normalizeOsPath(runtime.GOROOT()))
   128  
   129  				args = append(args,
   130  					"-test.skip=TestRenameCaseDifference/dir|"+
   131  						"TestDirFSPathsValid|TestDirFS|TestDevNullFile|"+
   132  						"TestOpenError|TestSymlinkWithTrailingSlash")
   133  			}
   134  			c = c.WithArgs(args...)
   135  
   136  			return bin, c, stdout, stderr, err
   137  		},
   138  	}
   139  )
   140  
   141  func runtBenches(b *testing.B, ctx context.Context, rc wazero.RuntimeConfig, tc testCase) {
   142  	cwd, _ := os.Getwd()
   143  	files, err := os.ReadDir(tc.dir)
   144  	require.NoError(b, err)
   145  	for _, f := range files {
   146  		fname := f.Name()
   147  		// Ensure we are on root dir.
   148  		err = os.Chdir(cwd)
   149  		require.NoError(b, err)
   150  
   151  		fpath := filepath.Join(cwd, tc.dir, fname)
   152  		bin, modCfg, stdout, stderr, err := tc.readTestCase(fpath, fname)
   153  		require.NoError(b, err)
   154  		if bin == nil {
   155  			continue
   156  		}
   157  
   158  		for _, compile := range []bool{false, true} {
   159  			if compile {
   160  				b.Run("Compile/"+fname, func(b *testing.B) {
   161  					b.ResetTimer()
   162  					for i := 0; i < b.N; i++ {
   163  						r := wazero.NewRuntimeWithConfig(ctx, rc)
   164  						_, err := r.CompileModule(ctx, bin)
   165  						require.NoError(b, err)
   166  						require.NoError(b, r.Close(ctx))
   167  					}
   168  				})
   169  			} else {
   170  				r := wazero.NewRuntimeWithConfig(ctx, rc)
   171  				wasi_snapshot_preview1.MustInstantiate(ctx, r)
   172  				b.Cleanup(func() { r.Close(ctx) })
   173  
   174  				cm, err := r.CompileModule(ctx, bin)
   175  				require.NoError(b, err)
   176  				b.Run("Run/"+fname, func(b *testing.B) {
   177  					b.ResetTimer()
   178  					for i := 0; i < b.N; i++ {
   179  						// Instantiate in the loop as _start cannot be called multiple times.
   180  						m, err := r.InstantiateModule(ctx, cm, modCfg)
   181  						requireZeroExitCode(b, err, stdout, stderr)
   182  						require.NoError(b, m.Close(ctx))
   183  					}
   184  				})
   185  			}
   186  		}
   187  	}
   188  }
   189  
   190  // Normalize an absolute path to a Unix-style path, regardless if it is a Windows path.
   191  func normalizeOsPath(path string) string {
   192  	// Remove volume name. This is '/' on *Nix and 'C:' (with C being any letter identifier).
   193  	root := filepath.VolumeName(path)
   194  	testdirnoprefix := path[len(root):]
   195  	// Normalizes all the path separators to a Unix separator.
   196  	testdirnormalized := strings.ReplaceAll(testdirnoprefix, string(os.PathSeparator), "/")
   197  	return testdirnormalized
   198  }
   199  
   200  func defaultModuleConfig() (c wazero.ModuleConfig, stdout, stderr *os.File) {
   201  	var err error
   202  	// Note: do not use os.Stdout or os.Stderr as they will mess up the `-bench` output to be fed to the benchstat tool.
   203  	stdout, err = os.CreateTemp("", "")
   204  	if err != nil {
   205  		panic(err)
   206  	}
   207  	stderr, err = os.CreateTemp("", "")
   208  	if err != nil {
   209  		panic(err)
   210  	}
   211  	c = wazero.NewModuleConfig().
   212  		WithSysNanosleep().
   213  		WithSysNanotime().
   214  		WithSysWalltime().
   215  		WithRandSource(rand.Reader).
   216  		// Some tests require Stdout and Stderr to be present.
   217  		WithStdout(stdout).
   218  		WithStderr(stderr)
   219  	return
   220  }
   221  
   222  func requireZeroExitCode(b *testing.B, err error, stdout, stderr *os.File) {
   223  	b.Helper()
   224  	if se, ok := err.(*sys.ExitError); ok {
   225  		if se.ExitCode() != 0 { // Don't err on success.
   226  			stdoutBytes, _ := io.ReadAll(stdout)
   227  			stderrBytes, _ := io.ReadAll(stderr)
   228  			require.NoError(b, err, "stdout: %s\nstderr: %s", string(stdoutBytes), string(stderrBytes))
   229  		}
   230  	} else if err != nil {
   231  		stdoutBytes, _ := io.ReadAll(stdout)
   232  		stderrBytes, _ := io.ReadAll(stderr)
   233  		require.NoError(b, err, "stdout: %s\nstderr: %s", string(stdoutBytes), string(stderrBytes))
   234  	}
   235  }
   236
View as plain text