runner.go

Documentation: github.com/BurntSushi/toml/internal/toml-test

     1  //go:generate ./gen-multi.py
     2  
     3  //go:build go1.16
     4  // +build go1.16
     5  
     6  package tomltest
     7  
     8  import (
     9  	"bytes"
    10  	"embed"
    11  	"encoding/json"
    12  	"errors"
    13  	"fmt"
    14  	"io/fs"
    15  	"os/exec"
    16  	"path/filepath"
    17  	"sort"
    18  	"strings"
    19  
    20  	"github.com/BurntSushi/toml"
    21  )
    22  
    23  type testType uint8
    24  
    25  const (
    26  	TypeValid testType = iota
    27  	TypeInvalid
    28  )
    29  
    30  //go:embed tests/*
    31  var embeddedTests embed.FS
    32  
    33  // EmbeddedTests are the tests embedded in toml-test, rooted to the "test/"
    34  // directory.
    35  func EmbeddedTests() fs.FS {
    36  	f, err := fs.Sub(embeddedTests, "tests")
    37  	if err != nil {
    38  		panic(err)
    39  	}
    40  	return f
    41  }
    42  
    43  // Runner runs a set of tests.
    44  //
    45  // The validity of the parameters is not checked extensively; the caller should
    46  // verify this if need be. See ./cmd/toml-test for an example.
    47  type Runner struct {
    48  	Files     fs.FS    // Test files.
    49  	Encoder   bool     // Are we testing an encoder?
    50  	RunTests  []string // Tests to run; run all if blank.
    51  	SkipTests []string // Tests to skip.
    52  	Parser    Parser   // Send data to a parser.
    53  	Version   string   // TOML version to run tests for.
    54  }
    55  
    56  // A Parser instance is used to call the TOML parser we test.
    57  //
    58  // By default this is done through an external command.
    59  type Parser interface {
    60  	// Encode a JSON string to TOML.
    61  	//
    62  	// The output is the TOML string; if outputIsError is true then it's assumed
    63  	// that an encoding error occurred.
    64  	//
    65  	// An error return should only be used in case an unrecoverable error
    66  	// occurred; failing to encode to TOML is not an error, but the encoder
    67  	// unexpectedly panicking is.
    68  	Encode(jsonInput string) (output string, outputIsError bool, err error)
    69  
    70  	// Decode a TOML string to JSON. The same semantics as Encode apply.
    71  	Decode(tomlInput string) (output string, outputIsError bool, err error)
    72  }
    73  
    74  // CommandParser calls an external command.
    75  type CommandParser struct {
    76  	fsys fs.FS
    77  	cmd  []string
    78  }
    79  
    80  // Tests are tests to run.
    81  type Tests struct {
    82  	Tests []Test
    83  
    84  	// Set when test are run.
    85  
    86  	Skipped, Passed, Failed int
    87  }
    88  
    89  // Result is the result of a single test.
    90  type Test struct {
    91  	Path string // Path of test, e.g. "valid/string-test"
    92  
    93  	// Set when a test is run.
    94  
    95  	Skipped          bool   // Skipped this test?
    96  	Failure          string // Failure message.
    97  	Key              string // TOML key the failure occured on; may be blank.
    98  	Encoder          bool   // Encoder test?
    99  	Input            string // The test case that we sent to the external program.
   100  	Output           string // Output from the external program.
   101  	Want             string // The output we want.
   102  	OutputFromStderr bool   // The Output came from stderr, not stdout.
   103  }
   104  
   105  // List all tests in Files for the current TOML version.
   106  func (r Runner) List() ([]string, error) {
   107  	if r.Version == "" {
   108  		r.Version = "1.0.0"
   109  	}
   110  	if _, ok := versions[r.Version]; !ok {
   111  		v := make([]string, 0, len(versions))
   112  		for k := range versions {
   113  			v = append(v, k)
   114  		}
   115  		sort.Strings(v)
   116  		return nil, fmt.Errorf("tomltest.Runner.Run: unknown version: %q (supported: \"%s\")",
   117  			r.Version, strings.Join(v, `", "`))
   118  	}
   119  
   120  	var (
   121  		v       = versions[r.Version]
   122  		exclude = make([]string, 0, 8)
   123  	)
   124  	for {
   125  		exclude = append(exclude, v.exclude...)
   126  		if v.inherit == "" {
   127  			break
   128  		}
   129  		v = versions[v.inherit]
   130  	}
   131  
   132  	ls := make([]string, 0, 256)
   133  	if err := r.findTOML("valid", &ls, exclude); err != nil {
   134  		return nil, fmt.Errorf("reading 'valid/' dir: %w", err)
   135  	}
   136  
   137  	d := "invalid" + map[bool]string{true: "-encoder", false: ""}[r.Encoder]
   138  	if err := r.findTOML(d, &ls, exclude); err != nil {
   139  		return nil, fmt.Errorf("reading %q dir: %w", d, err)
   140  	}
   141  
   142  	return ls, nil
   143  }
   144  
   145  // Run all tests listed in t.RunTests.
   146  //
   147  // TODO: give option to:
   148  // - Run all tests with \n replaced with \r\n
   149  // - Run all tests with EOL removed
   150  // - Run all tests with '# comment' appended to every line.
   151  func (r Runner) Run() (Tests, error) {
   152  	skipped, err := r.findTests()
   153  	if err != nil {
   154  		return Tests{}, fmt.Errorf("tomltest.Runner.Run: %w", err)
   155  	}
   156  
   157  	tests := Tests{Tests: make([]Test, 0, len(r.RunTests)), Skipped: skipped}
   158  	for _, p := range r.RunTests {
   159  		if r.hasSkip(p) {
   160  			tests.Skipped++
   161  			tests.Tests = append(tests.Tests, Test{Path: p, Skipped: true, Encoder: r.Encoder})
   162  			continue
   163  		}
   164  
   165  		t := Test{Path: p, Encoder: r.Encoder}.Run(r.Parser, r.Files)
   166  		tests.Tests = append(tests.Tests, t)
   167  
   168  		if t.Failed() {
   169  			tests.Failed++
   170  		} else {
   171  			tests.Passed++
   172  		}
   173  	}
   174  
   175  	return tests, nil
   176  }
   177  
   178  // find all TOML files in 'path' relative to the test directory.
   179  func (r Runner) findTOML(path string, appendTo *[]string, exclude []string) error {
   180  	// It's okay if the directory doesn't exist.
   181  	if _, err := fs.Stat(r.Files, path); errors.Is(err, fs.ErrNotExist) {
   182  		return nil
   183  	}
   184  
   185  	return fs.WalkDir(r.Files, path, func(path string, d fs.DirEntry, err error) error {
   186  		if err != nil {
   187  			return err
   188  		}
   189  		if d.IsDir() || !strings.HasSuffix(path, ".toml") {
   190  			return nil
   191  		}
   192  
   193  		path = strings.TrimSuffix(path, ".toml")
   194  		for _, e := range exclude {
   195  			if ok, _ := filepath.Match(e, path); ok {
   196  				return nil
   197  			}
   198  		}
   199  
   200  		*appendTo = append(*appendTo, path)
   201  		return nil
   202  	})
   203  }
   204  
   205  // Expand RunTest glob patterns, or return all tests if RunTests if empty.
   206  func (r *Runner) findTests() (int, error) {
   207  	ls, err := r.List()
   208  	if err != nil {
   209  		return 0, err
   210  	}
   211  
   212  	var skip int
   213  
   214  	if len(r.RunTests) == 0 {
   215  		r.RunTests = ls
   216  	} else {
   217  		run := make([]string, 0, len(r.RunTests))
   218  		for _, l := range ls {
   219  			for _, r := range r.RunTests {
   220  				if m, _ := filepath.Match(r, l); m {
   221  					run = append(run, l)
   222  					break
   223  				}
   224  			}
   225  		}
   226  		r.RunTests, skip = run, len(ls)-len(run)
   227  	}
   228  
   229  	// Expand invalid tests ending in ".multi.toml"
   230  	expanded := make([]string, 0, len(r.RunTests))
   231  	for _, path := range r.RunTests {
   232  		if !strings.HasSuffix(path, ".multi") {
   233  			expanded = append(expanded, path)
   234  			continue
   235  		}
   236  
   237  		d, err := fs.ReadFile(r.Files, path+".toml")
   238  		if err != nil {
   239  			return 0, err
   240  		}
   241  
   242  		fmt.Println(string(d))
   243  	}
   244  	r.RunTests = expanded
   245  
   246  	return skip, nil
   247  }
   248  
   249  func (r Runner) hasSkip(path string) bool {
   250  	for _, s := range r.SkipTests {
   251  		if m, _ := filepath.Match(s, path); m {
   252  			return true
   253  		}
   254  	}
   255  	return false
   256  }
   257  
   258  func (c CommandParser) Encode(input string) (output string, outputIsError bool, err error) {
   259  	stdout, stderr := new(bytes.Buffer), new(bytes.Buffer)
   260  	cmd := exec.Command(c.cmd[0])
   261  	cmd.Args = c.cmd
   262  	cmd.Stdin, cmd.Stdout, cmd.Stderr = strings.NewReader(input), stdout, stderr
   263  
   264  	err = cmd.Run()
   265  	if err != nil {
   266  		eErr := &exec.ExitError{}
   267  		if errors.As(err, &eErr) {
   268  			fmt.Fprintf(stderr, "\nExit %d\n", eErr.ProcessState.ExitCode())
   269  			err = nil
   270  		}
   271  	}
   272  
   273  	if stderr.Len() > 0 {
   274  		return strings.TrimSpace(stderr.String()) + "\n", true, err
   275  	}
   276  	return strings.TrimSpace(stdout.String()) + "\n", false, err
   277  }
   278  func NewCommandParser(fsys fs.FS, cmd []string) CommandParser     { return CommandParser{fsys, cmd} }
   279  func (c CommandParser) Decode(input string) (string, bool, error) { return c.Encode(input) }
   280  
   281  // Run this test.
   282  func (t Test) Run(p Parser, fsys fs.FS) Test {
   283  	if t.Type() == TypeInvalid {
   284  		return t.runInvalid(p, fsys)
   285  	}
   286  	return t.runValid(p, fsys)
   287  }
   288  
   289  func (t Test) runInvalid(p Parser, fsys fs.FS) Test {
   290  	var err error
   291  	_, t.Input, err = t.ReadInput(fsys)
   292  	if err != nil {
   293  		return t.bug(err.Error())
   294  	}
   295  
   296  	if t.Encoder {
   297  		t.Output, t.OutputFromStderr, err = p.Encode(t.Input)
   298  	} else {
   299  		t.Output, t.OutputFromStderr, err = p.Decode(t.Input)
   300  	}
   301  	if err != nil {
   302  		return t.fail(err.Error())
   303  	}
   304  	if !t.OutputFromStderr {
   305  		return t.fail("Expected an error, but no error was reported.")
   306  	}
   307  	return t
   308  }
   309  
   310  func (t Test) runValid(p Parser, fsys fs.FS) Test {
   311  	var err error
   312  	_, t.Input, err = t.ReadInput(fsys)
   313  	if err != nil {
   314  		return t.bug(err.Error())
   315  	}
   316  
   317  	if t.Encoder {
   318  		t.Output, t.OutputFromStderr, err = p.Encode(t.Input)
   319  	} else {
   320  		t.Output, t.OutputFromStderr, err = p.Decode(t.Input)
   321  	}
   322  	if err != nil {
   323  		return t.fail(err.Error())
   324  	}
   325  	if t.OutputFromStderr {
   326  		return t.fail(t.Output)
   327  	}
   328  	if t.Output == "" {
   329  		// Special case: we expect an empty output here.
   330  		if t.Path != "valid/empty-file" {
   331  			return t.fail("stdout is empty")
   332  		}
   333  	}
   334  
   335  	// Compare for encoder test
   336  	if t.Encoder {
   337  		want, err := t.ReadWantTOML(fsys)
   338  		if err != nil {
   339  			return t.bug(err.Error())
   340  		}
   341  		var have interface{}
   342  		if _, err := toml.Decode(t.Output, &have); err != nil {
   343  			//return t.fail("decode TOML from encoder %q:\n  %s", cmd, err)
   344  			return t.fail("decode TOML from encoder:\n  %s", err)
   345  		}
   346  		return t.CompareTOML(want, have)
   347  	}
   348  
   349  	// Compare for decoder test
   350  	want, err := t.ReadWantJSON(fsys)
   351  	if err != nil {
   352  		return t.fail(err.Error())
   353  	}
   354  
   355  	var have interface{}
   356  	if err := json.Unmarshal([]byte(t.Output), &have); err != nil {
   357  		return t.fail("decode JSON output from parser:\n  %s", err)
   358  	}
   359  
   360  	return t.CompareJSON(want, have)
   361  }
   362  
   363  // ReadInput reads the file sent to the encoder.
   364  func (t Test) ReadInput(fsys fs.FS) (path, data string, err error) {
   365  	path = t.Path + map[bool]string{true: ".json", false: ".toml"}[t.Encoder]
   366  	d, err := fs.ReadFile(fsys, path)
   367  	if err != nil {
   368  		return path, "", err
   369  	}
   370  	return path, string(d), nil
   371  }
   372  
   373  func (t Test) ReadWant(fsys fs.FS) (path, data string, err error) {
   374  	if t.Type() == TypeInvalid {
   375  		panic("testoml.Test.ReadWant: invalid tests do not have a 'correct' version")
   376  	}
   377  
   378  	path = t.Path + map[bool]string{true: ".toml", false: ".json"}[t.Encoder]
   379  	d, err := fs.ReadFile(fsys, path)
   380  	if err != nil {
   381  		return path, "", err
   382  	}
   383  	return path, string(d), nil
   384  }
   385  
   386  func (t *Test) ReadWantJSON(fsys fs.FS) (v interface{}, err error) {
   387  	var path string
   388  	path, t.Want, err = t.ReadWant(fsys)
   389  	if err != nil {
   390  		return nil, err
   391  	}
   392  
   393  	if err := json.Unmarshal([]byte(t.Want), &v); err != nil {
   394  		return nil, fmt.Errorf("decode JSON file %q:\n  %s", path, err)
   395  	}
   396  	return v, nil
   397  }
   398  func (t *Test) ReadWantTOML(fsys fs.FS) (v interface{}, err error) {
   399  	var path string
   400  	path, t.Want, err = t.ReadWant(fsys)
   401  	if err != nil {
   402  		return nil, err
   403  	}
   404  	_, err = toml.Decode(t.Want, &v)
   405  	if err != nil {
   406  		return nil, fmt.Errorf("could not decode TOML file %q:\n  %s", path, err)
   407  	}
   408  	return v, nil
   409  }
   410  
   411  // Test type: "valid", "invalid"
   412  func (t Test) Type() testType {
   413  	if strings.HasPrefix(t.Path, "invalid") {
   414  		return TypeInvalid
   415  	}
   416  	return TypeValid
   417  }
   418  
   419  func (t Test) fail(format string, v ...interface{}) Test {
   420  	t.Failure = fmt.Sprintf(format, v...)
   421  	return t
   422  }
   423  func (t Test) bug(format string, v ...interface{}) Test {
   424  	return t.fail("BUG IN TEST CASE: "+format, v...)
   425  }
   426  
   427  func (t Test) Failed() bool { return t.Failure != "" }
   428
View as plain text