bench_test.go

Documentation: go.starlark.net/starlark

     1  // Copyright 2018 The Bazel Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package starlark_test
     6  
     7  import (
     8  	"bytes"
     9  	"fmt"
    10  	"io/ioutil"
    11  	"path/filepath"
    12  	"strings"
    13  	"testing"
    14  
    15  	"go.starlark.net/lib/json"
    16  	"go.starlark.net/starlark"
    17  	"go.starlark.net/starlarktest"
    18  )
    19  
    20  func BenchmarkStarlark(b *testing.B) {
    21  	defer setOptions("")
    22  
    23  	starlark.Universe["json"] = json.Module
    24  
    25  	testdata := starlarktest.DataFile("starlark", ".")
    26  	thread := new(starlark.Thread)
    27  	for _, file := range []string{
    28  		"testdata/benchmark.star",
    29  		// ...
    30  	} {
    31  
    32  		filename := filepath.Join(testdata, file)
    33  
    34  		src, err := ioutil.ReadFile(filename)
    35  		if err != nil {
    36  			b.Error(err)
    37  			continue
    38  		}
    39  		setOptions(string(src))
    40  
    41  		// Evaluate the file once.
    42  		globals, err := starlark.ExecFile(thread, filename, src, nil)
    43  		if err != nil {
    44  			reportEvalError(b, err)
    45  		}
    46  
    47  		// Repeatedly call each global function named bench_* as a benchmark.
    48  		for _, name := range globals.Keys() {
    49  			value := globals[name]
    50  			if fn, ok := value.(*starlark.Function); ok && strings.HasPrefix(name, "bench_") {
    51  				b.Run(name, func(b *testing.B) {
    52  					_, err := starlark.Call(thread, fn, starlark.Tuple{benchmark{b}}, nil)
    53  					if err != nil {
    54  						reportEvalError(b, err)
    55  					}
    56  				})
    57  			}
    58  		}
    59  	}
    60  }
    61  
    62  // A benchmark is passed to each bench_xyz(b) function in a bench_*.star file.
    63  // It provides b.n, the number of iterations that must be executed by the function,
    64  // which is typically of the form:
    65  //
    66  //   def bench_foo(b):
    67  //      for _ in range(b.n):
    68  //         ...work...
    69  //
    70  // It also provides stop, start, and restart methods to stop the clock in case
    71  // there is significant set-up work that should not count against the measured
    72  // operation.
    73  //
    74  // (This interface is inspired by Go's testing.B, and is also implemented
    75  // by the java.starlark.net implementation; see
    76  // https://github.com/bazelbuild/starlark/pull/75#pullrequestreview-275604129.)
    77  type benchmark struct {
    78  	b *testing.B
    79  }
    80  
    81  func (benchmark) Freeze()               {}
    82  func (benchmark) Truth() starlark.Bool  { return true }
    83  func (benchmark) Type() string          { return "benchmark" }
    84  func (benchmark) String() string        { return "<benchmark>" }
    85  func (benchmark) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable: benchmark") }
    86  func (benchmark) AttrNames() []string   { return []string{"n", "restart", "start", "stop"} }
    87  func (b benchmark) Attr(name string) (starlark.Value, error) {
    88  	switch name {
    89  	case "n":
    90  		return starlark.MakeInt(b.b.N), nil
    91  	case "restart":
    92  		return benchmarkRestart.BindReceiver(b), nil
    93  	case "start":
    94  		return benchmarkStart.BindReceiver(b), nil
    95  	case "stop":
    96  		return benchmarkStop.BindReceiver(b), nil
    97  	}
    98  	return nil, nil
    99  }
   100  
   101  var (
   102  	benchmarkRestart = starlark.NewBuiltin("restart", benchmarkRestartImpl)
   103  	benchmarkStart   = starlark.NewBuiltin("start", benchmarkStartImpl)
   104  	benchmarkStop    = starlark.NewBuiltin("stop", benchmarkStopImpl)
   105  )
   106  
   107  func benchmarkRestartImpl(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
   108  	b.Receiver().(benchmark).b.ResetTimer()
   109  	return starlark.None, nil
   110  }
   111  
   112  func benchmarkStartImpl(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
   113  	b.Receiver().(benchmark).b.StartTimer()
   114  	return starlark.None, nil
   115  }
   116  
   117  func benchmarkStopImpl(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
   118  	b.Receiver().(benchmark).b.StopTimer()
   119  	return starlark.None, nil
   120  }
   121  
   122  // BenchmarkProgram measures operations relevant to compiled programs.
   123  // TODO(adonovan): use a bigger testdata program.
   124  func BenchmarkProgram(b *testing.B) {
   125  	// Measure time to read a source file (approx 600us but depends on hardware and file system).
   126  	filename := starlarktest.DataFile("starlark", "testdata/paths.star")
   127  	var src []byte
   128  	b.Run("read", func(b *testing.B) {
   129  		for i := 0; i < b.N; i++ {
   130  			var err error
   131  			src, err = ioutil.ReadFile(filename)
   132  			if err != nil {
   133  				b.Fatal(err)
   134  			}
   135  		}
   136  	})
   137  
   138  	// Measure time to turn a source filename into a compiled program (approx 450us).
   139  	var prog *starlark.Program
   140  	b.Run("compile", func(b *testing.B) {
   141  		for i := 0; i < b.N; i++ {
   142  			var err error
   143  			_, prog, err = starlark.SourceProgram(filename, src, starlark.StringDict(nil).Has)
   144  			if err != nil {
   145  				b.Fatal(err)
   146  			}
   147  		}
   148  	})
   149  
   150  	// Measure time to encode a compiled program to a memory buffer
   151  	// (approx 20us; was 75-120us with gob encoding).
   152  	var out bytes.Buffer
   153  	b.Run("encode", func(b *testing.B) {
   154  		for i := 0; i < b.N; i++ {
   155  			out.Reset()
   156  			if err := prog.Write(&out); err != nil {
   157  				b.Fatal(err)
   158  			}
   159  		}
   160  	})
   161  
   162  	// Measure time to decode a compiled program from a memory buffer
   163  	// (approx 20us; was 135-250us with gob encoding)
   164  	b.Run("decode", func(b *testing.B) {
   165  		for i := 0; i < b.N; i++ {
   166  			in := bytes.NewReader(out.Bytes())
   167  			if _, err := starlark.CompiledProgram(in); err != nil {
   168  				b.Fatal(err)
   169  			}
   170  		}
   171  	})
   172  }
   173
View as plain text