Source file src/go.starlark.net/syntax/parse_test.go

Documentation: go.starlark.net/syntax

     1  // Copyright 2017 The Bazel Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package syntax_test
     6  
     7  import (
     8  	"bufio"
     9  	"bytes"
    10  	"fmt"
    11  	"go/build"
    12  	"io/ioutil"
    13  	"path/filepath"
    14  	"reflect"
    15  	"strings"
    16  	"testing"
    17  
    18  	"go.starlark.net/internal/chunkedfile"
    19  	"go.starlark.net/starlarktest"
    20  	"go.starlark.net/syntax"
    21  )
    22  
    23  func TestExprParseTrees(t *testing.T) {
    24  	for _, test := range []struct {
    25  		input, want string
    26  	}{
    27  		{`print(1)`,
    28  			`(CallExpr Fn=print Args=(1))`},
    29  		{"print(1)\n",
    30  			`(CallExpr Fn=print Args=(1))`},
    31  		{`x + 1`,
    32  			`(BinaryExpr X=x Op=+ Y=1)`},
    33  		{`[x for x in y]`,
    34  			`(Comprehension Body=x Clauses=((ForClause Vars=x X=y)))`},
    35  		{`[x for x in (a if b else c)]`,
    36  			`(Comprehension Body=x Clauses=((ForClause Vars=x X=(ParenExpr X=(CondExpr Cond=b True=a False=c)))))`},
    37  		{`x[i].f(42)`,
    38  			`(CallExpr Fn=(DotExpr X=(IndexExpr X=x Y=i) Name=f) Args=(42))`},
    39  		{`x.f()`,
    40  			`(CallExpr Fn=(DotExpr X=x Name=f))`},
    41  		{`x+y*z`,
    42  			`(BinaryExpr X=x Op=+ Y=(BinaryExpr X=y Op=* Y=z))`},
    43  		{`x%y-z`,
    44  			`(BinaryExpr X=(BinaryExpr X=x Op=% Y=y) Op=- Y=z)`},
    45  		{`a + b not in c`,
    46  			`(BinaryExpr X=(BinaryExpr X=a Op=+ Y=b) Op=not in Y=c)`},
    47  		{`lambda x, *args, **kwargs: None`,
    48  			`(LambdaExpr Params=(x (UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)) Body=None)`},
    49  		{`{"one": 1}`,
    50  			`(DictExpr List=((DictEntry Key="one" Value=1)))`},
    51  		{`a[i]`,
    52  			`(IndexExpr X=a Y=i)`},
    53  		{`a[i:]`,
    54  			`(SliceExpr X=a Lo=i)`},
    55  		{`a[:j]`,
    56  			`(SliceExpr X=a Hi=j)`},
    57  		{`a[::]`,
    58  			`(SliceExpr X=a)`},
    59  		{`a[::k]`,
    60  			`(SliceExpr X=a Step=k)`},
    61  		{`[]`,
    62  			`(ListExpr)`},
    63  		{`[1]`,
    64  			`(ListExpr List=(1))`},
    65  		{`[1,]`,
    66  			`(ListExpr List=(1))`},
    67  		{`[1, 2]`,
    68  			`(ListExpr List=(1 2))`},
    69  		{`()`,
    70  			`(TupleExpr)`},
    71  		{`(4,)`,
    72  			`(ParenExpr X=(TupleExpr List=(4)))`},
    73  		{`(4)`,
    74  			`(ParenExpr X=4)`},
    75  		{`(4, 5)`,
    76  			`(ParenExpr X=(TupleExpr List=(4 5)))`},
    77  		{`1, 2, 3`,
    78  			`(TupleExpr List=(1 2 3))`},
    79  		{`1, 2,`,
    80  			`unparenthesized tuple with trailing comma`},
    81  		{`{}`,
    82  			`(DictExpr)`},
    83  		{`{"a": 1}`,
    84  			`(DictExpr List=((DictEntry Key="a" Value=1)))`},
    85  		{`{"a": 1,}`,
    86  			`(DictExpr List=((DictEntry Key="a" Value=1)))`},
    87  		{`{"a": 1, "b": 2}`,
    88  			`(DictExpr List=((DictEntry Key="a" Value=1) (DictEntry Key="b" Value=2)))`},
    89  		{`{x: y for (x, y) in z}`,
    90  			`(Comprehension Curly Body=(DictEntry Key=x Value=y) Clauses=((ForClause Vars=(ParenExpr X=(TupleExpr List=(x y))) X=z)))`},
    91  		{`{x: y for a in b if c}`,
    92  			`(Comprehension Curly Body=(DictEntry Key=x Value=y) Clauses=((ForClause Vars=a X=b) (IfClause Cond=c)))`},
    93  		{`-1 + +2`,
    94  			`(BinaryExpr X=(UnaryExpr Op=- X=1) Op=+ Y=(UnaryExpr Op=+ X=2))`},
    95  		{`"foo" + "bar"`,
    96  			`(BinaryExpr X="foo" Op=+ Y="bar")`},
    97  		{`-1 * 2`, // prec(unary -) > prec(binary *)
    98  			`(BinaryExpr X=(UnaryExpr Op=- X=1) Op=* Y=2)`},
    99  		{`-x[i]`, // prec(unary -) < prec(x[i])
   100  			`(UnaryExpr Op=- X=(IndexExpr X=x Y=i))`},
   101  		{`a | b & c | d`, // prec(|) < prec(&)
   102  			`(BinaryExpr X=(BinaryExpr X=a Op=| Y=(BinaryExpr X=b Op=& Y=c)) Op=| Y=d)`},
   103  		{`a or b and c or d`,
   104  			`(BinaryExpr X=(BinaryExpr X=a Op=or Y=(BinaryExpr X=b Op=and Y=c)) Op=or Y=d)`},
   105  		{`a and b or c and d`,
   106  			`(BinaryExpr X=(BinaryExpr X=a Op=and Y=b) Op=or Y=(BinaryExpr X=c Op=and Y=d))`},
   107  		{`f(1, x=y)`,
   108  			`(CallExpr Fn=f Args=(1 (BinaryExpr X=x Op== Y=y)))`},
   109  		{`f(*args, **kwargs)`,
   110  			`(CallExpr Fn=f Args=((UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)))`},
   111  		{`lambda *args, *, x=1, **kwargs: 0`,
   112  			`(LambdaExpr Params=((UnaryExpr Op=* X=args) (UnaryExpr Op=*) (BinaryExpr X=x Op== Y=1) (UnaryExpr Op=** X=kwargs)) Body=0)`},
   113  		{`lambda *, a, *b: 0`,
   114  			`(LambdaExpr Params=((UnaryExpr Op=*) a (UnaryExpr Op=* X=b)) Body=0)`},
   115  		{`a if b else c`,
   116  			`(CondExpr Cond=b True=a False=c)`},
   117  		{`a and not b`,
   118  			`(BinaryExpr X=a Op=and Y=(UnaryExpr Op=not X=b))`},
   119  		{`[e for x in y if cond1 if cond2]`,
   120  			`(Comprehension Body=e Clauses=((ForClause Vars=x X=y) (IfClause Cond=cond1) (IfClause Cond=cond2)))`}, // github.com/google/skylark/issues/53
   121  	} {
   122  		e, err := syntax.ParseExpr("foo.star", test.input, 0)
   123  		var got string
   124  		if err != nil {
   125  			got = stripPos(err)
   126  		} else {
   127  			got = treeString(e)
   128  		}
   129  		if test.want != got {
   130  			t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want)
   131  		}
   132  	}
   133  }
   134  
   135  func TestStmtParseTrees(t *testing.T) {
   136  	for _, test := range []struct {
   137  		input, want string
   138  	}{
   139  		{`print(1)`,
   140  			`(ExprStmt X=(CallExpr Fn=print Args=(1)))`},
   141  		{`return 1, 2`,
   142  			`(ReturnStmt Result=(TupleExpr List=(1 2)))`},
   143  		{`return`,
   144  			`(ReturnStmt)`},
   145  		{`for i in "abc": break`,
   146  			`(ForStmt Vars=i X="abc" Body=((BranchStmt Token=break)))`},
   147  		{`for i in "abc": continue`,
   148  			`(ForStmt Vars=i X="abc" Body=((BranchStmt Token=continue)))`},
   149  		{`for x, y in z: pass`,
   150  			`(ForStmt Vars=(TupleExpr List=(x y)) X=z Body=((BranchStmt Token=pass)))`},
   151  		{`if True: pass`,
   152  			`(IfStmt Cond=True True=((BranchStmt Token=pass)))`},
   153  		{`if True: break`,
   154  			`(IfStmt Cond=True True=((BranchStmt Token=break)))`},
   155  		{`if True: continue`,
   156  			`(IfStmt Cond=True True=((BranchStmt Token=continue)))`},
   157  		{`if True: pass
   158  else:
   159  	pass`,
   160  			`(IfStmt Cond=True True=((BranchStmt Token=pass)) False=((BranchStmt Token=pass)))`},
   161  		{"if a: pass\nelif b: pass\nelse: pass",
   162  			`(IfStmt Cond=a True=((BranchStmt Token=pass)) False=((IfStmt Cond=b True=((BranchStmt Token=pass)) False=((BranchStmt Token=pass)))))`},
   163  		{`x, y = 1, 2`,
   164  			`(AssignStmt Op== LHS=(TupleExpr List=(x y)) RHS=(TupleExpr List=(1 2)))`},
   165  		{`x[i] = 1`,
   166  			`(AssignStmt Op== LHS=(IndexExpr X=x Y=i) RHS=1)`},
   167  		{`x.f = 1`,
   168  			`(AssignStmt Op== LHS=(DotExpr X=x Name=f) RHS=1)`},
   169  		{`(x, y) = 1`,
   170  			`(AssignStmt Op== LHS=(ParenExpr X=(TupleExpr List=(x y))) RHS=1)`},
   171  		{`load("", "a", b="c")`,
   172  			`(LoadStmt Module="" From=(a c) To=(a b))`},
   173  		{`if True: load("", "a", b="c")`, // load needn't be at toplevel
   174  			`(IfStmt Cond=True True=((LoadStmt Module="" From=(a c) To=(a b))))`},
   175  		{`def f(x, *args, **kwargs):
   176  	pass`,
   177  			`(DefStmt Name=f Params=(x (UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)) Body=((BranchStmt Token=pass)))`},
   178  		{`def f(**kwargs, *args): pass`,
   179  			`(DefStmt Name=f Params=((UnaryExpr Op=** X=kwargs) (UnaryExpr Op=* X=args)) Body=((BranchStmt Token=pass)))`},
   180  		{`def f(a, b, c=d): pass`,
   181  			`(DefStmt Name=f Params=(a b (BinaryExpr X=c Op== Y=d)) Body=((BranchStmt Token=pass)))`},
   182  		{`def f(a, b=c, d): pass`,
   183  			`(DefStmt Name=f Params=(a (BinaryExpr X=b Op== Y=c) d) Body=((BranchStmt Token=pass)))`}, // TODO(adonovan): fix this
   184  		{`def f():
   185  	def g():
   186  		pass
   187  	pass
   188  def h():
   189  	pass`,
   190  			`(DefStmt Name=f Body=((DefStmt Name=g Body=((BranchStmt Token=pass))) (BranchStmt Token=pass)))`},
   191  		{"f();g()",
   192  			`(ExprStmt X=(CallExpr Fn=f))`},
   193  		{"f();",
   194  			`(ExprStmt X=(CallExpr Fn=f))`},
   195  		{"f();g()\n",
   196  			`(ExprStmt X=(CallExpr Fn=f))`},
   197  		{"f();\n",
   198  			`(ExprStmt X=(CallExpr Fn=f))`},
   199  	} {
   200  		f, err := syntax.Parse("foo.star", test.input, 0)
   201  		if err != nil {
   202  			t.Errorf("parse `%s` failed: %v", test.input, stripPos(err))
   203  			continue
   204  		}
   205  		if got := treeString(f.Stmts[0]); test.want != got {
   206  			t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want)
   207  		}
   208  	}
   209  }
   210  
   211  // TestFileParseTrees tests sequences of statements, and particularly
   212  // handling of indentation, newlines, line continuations, and blank lines.
   213  func TestFileParseTrees(t *testing.T) {
   214  	for _, test := range []struct {
   215  		input, want string
   216  	}{
   217  		{`x = 1
   218  print(x)`,
   219  			`(AssignStmt Op== LHS=x RHS=1)
   220  (ExprStmt X=(CallExpr Fn=print Args=(x)))`},
   221  		{"if cond:\n\tpass",
   222  			`(IfStmt Cond=cond True=((BranchStmt Token=pass)))`},
   223  		{"if cond:\n\tpass\nelse:\n\tpass",
   224  			`(IfStmt Cond=cond True=((BranchStmt Token=pass)) False=((BranchStmt Token=pass)))`},
   225  		{`def f():
   226  	pass
   227  pass
   228  
   229  pass`,
   230  			`(DefStmt Name=f Body=((BranchStmt Token=pass)))
   231  (BranchStmt Token=pass)
   232  (BranchStmt Token=pass)`},
   233  		{`pass; pass`,
   234  			`(BranchStmt Token=pass)
   235  (BranchStmt Token=pass)`},
   236  		{"pass\npass",
   237  			`(BranchStmt Token=pass)
   238  (BranchStmt Token=pass)`},
   239  		{"pass\n\npass",
   240  			`(BranchStmt Token=pass)
   241  (BranchStmt Token=pass)`},
   242  		{`x = (1 +
   243  2)`,
   244  			`(AssignStmt Op== LHS=x RHS=(ParenExpr X=(BinaryExpr X=1 Op=+ Y=2)))`},
   245  		{`x = 1 \
   246  + 2`,
   247  			`(AssignStmt Op== LHS=x RHS=(BinaryExpr X=1 Op=+ Y=2))`},
   248  	} {
   249  		f, err := syntax.Parse("foo.star", test.input, 0)
   250  		if err != nil {
   251  			t.Errorf("parse `%s` failed: %v", test.input, stripPos(err))
   252  			continue
   253  		}
   254  		var buf bytes.Buffer
   255  		for i, stmt := range f.Stmts {
   256  			if i > 0 {
   257  				buf.WriteByte('\n')
   258  			}
   259  			writeTree(&buf, reflect.ValueOf(stmt))
   260  		}
   261  		if got := buf.String(); test.want != got {
   262  			t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want)
   263  		}
   264  	}
   265  }
   266  
   267  // TestCompoundStmt tests handling of REPL-style compound statements.
   268  func TestCompoundStmt(t *testing.T) {
   269  	for _, test := range []struct {
   270  		input, want string
   271  	}{
   272  		// blank lines
   273  		{"\n",
   274  			``},
   275  		{"   \n",
   276  			``},
   277  		{"# comment\n",
   278  			``},
   279  		// simple statement
   280  		{"1\n",
   281  			`(ExprStmt X=1)`},
   282  		{"print(1)\n",
   283  			`(ExprStmt X=(CallExpr Fn=print Args=(1)))`},
   284  		{"1;2;3;\n",
   285  			`(ExprStmt X=1)(ExprStmt X=2)(ExprStmt X=3)`},
   286  		{"f();g()\n",
   287  			`(ExprStmt X=(CallExpr Fn=f))(ExprStmt X=(CallExpr Fn=g))`},
   288  		{"f();\n",
   289  			`(ExprStmt X=(CallExpr Fn=f))`},
   290  		{"f(\n\n\n\n\n\n\n)\n",
   291  			`(ExprStmt X=(CallExpr Fn=f))`},
   292  		// complex statements
   293  		{"def f():\n  pass\n\n",
   294  			`(DefStmt Name=f Body=((BranchStmt Token=pass)))`},
   295  		{"if cond:\n  pass\n\n",
   296  			`(IfStmt Cond=cond True=((BranchStmt Token=pass)))`},
   297  		// Even as a 1-liner, the following blank line is required.
   298  		{"if cond: pass\n\n",
   299  			`(IfStmt Cond=cond True=((BranchStmt Token=pass)))`},
   300  		// github.com/google/starlark-go/issues/121
   301  		{"a; b; c\n",
   302  			`(ExprStmt X=a)(ExprStmt X=b)(ExprStmt X=c)`},
   303  		{"a; b c\n",
   304  			`invalid syntax`},
   305  	} {
   306  
   307  		// Fake readline input from string.
   308  		// The ! suffix, which would cause a parse error,
   309  		// tests that the parser doesn't read more than necessary.
   310  		sc := bufio.NewScanner(strings.NewReader(test.input + "!"))
   311  		readline := func() ([]byte, error) {
   312  			if sc.Scan() {
   313  				return []byte(sc.Text() + "\n"), nil
   314  			}
   315  			return nil, sc.Err()
   316  		}
   317  
   318  		var got string
   319  		f, err := syntax.ParseCompoundStmt("foo.star", readline)
   320  		if err != nil {
   321  			got = stripPos(err)
   322  		} else {
   323  			for _, stmt := range f.Stmts {
   324  				got += treeString(stmt)
   325  			}
   326  		}
   327  		if test.want != got {
   328  			t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want)
   329  		}
   330  	}
   331  }
   332  
   333  func stripPos(err error) string {
   334  	s := err.Error()
   335  	if i := strings.Index(s, ": "); i >= 0 {
   336  		s = s[i+len(": "):] // strip file:line:col
   337  	}
   338  	return s
   339  }
   340  
   341  // treeString prints a syntax node as a parenthesized tree.
   342  // Idents are printed as foo and Literals as "foo" or 42.
   343  // Structs are printed as (type name=value ...).
   344  // Only non-empty fields are shown.
   345  func treeString(n syntax.Node) string {
   346  	var buf bytes.Buffer
   347  	writeTree(&buf, reflect.ValueOf(n))
   348  	return buf.String()
   349  }
   350  
   351  func writeTree(out *bytes.Buffer, x reflect.Value) {
   352  	switch x.Kind() {
   353  	case reflect.String, reflect.Int, reflect.Bool:
   354  		fmt.Fprintf(out, "%v", x.Interface())
   355  	case reflect.Ptr, reflect.Interface:
   356  		if elem := x.Elem(); elem.Kind() == 0 {
   357  			out.WriteString("nil")
   358  		} else {
   359  			writeTree(out, elem)
   360  		}
   361  	case reflect.Struct:
   362  		switch v := x.Interface().(type) {
   363  		case syntax.Literal:
   364  			switch v.Token {
   365  			case syntax.STRING:
   366  				fmt.Fprintf(out, "%q", v.Value)
   367  			case syntax.BYTES:
   368  				fmt.Fprintf(out, "b%q", v.Value)
   369  			case syntax.INT:
   370  				fmt.Fprintf(out, "%d", v.Value)
   371  			}
   372  			return
   373  		case syntax.Ident:
   374  			out.WriteString(v.Name)
   375  			return
   376  		}
   377  		fmt.Fprintf(out, "(%s", strings.TrimPrefix(x.Type().String(), "syntax."))
   378  		for i, n := 0, x.NumField(); i < n; i++ {
   379  			f := x.Field(i)
   380  			if f.Type() == reflect.TypeOf(syntax.Position{}) {
   381  				continue // skip positions
   382  			}
   383  			name := x.Type().Field(i).Name
   384  			if name == "commentsRef" {
   385  				continue // skip comments fields
   386  			}
   387  			if f.Type() == reflect.TypeOf(syntax.Token(0)) {
   388  				fmt.Fprintf(out, " %s=%s", name, f.Interface())
   389  				continue
   390  			}
   391  
   392  			switch f.Kind() {
   393  			case reflect.Slice:
   394  				if n := f.Len(); n > 0 {
   395  					fmt.Fprintf(out, " %s=(", name)
   396  					for i := 0; i < n; i++ {
   397  						if i > 0 {
   398  							out.WriteByte(' ')
   399  						}
   400  						writeTree(out, f.Index(i))
   401  					}
   402  					out.WriteByte(')')
   403  				}
   404  				continue
   405  			case reflect.Ptr, reflect.Interface:
   406  				if f.IsNil() {
   407  					continue
   408  				}
   409  			case reflect.Int:
   410  				if f.Int() != 0 {
   411  					fmt.Fprintf(out, " %s=%d", name, f.Int())
   412  				}
   413  				continue
   414  			case reflect.Bool:
   415  				if f.Bool() {
   416  					fmt.Fprintf(out, " %s", name)
   417  				}
   418  				continue
   419  			}
   420  			fmt.Fprintf(out, " %s=", name)
   421  			writeTree(out, f)
   422  		}
   423  		fmt.Fprintf(out, ")")
   424  	default:
   425  		fmt.Fprintf(out, "%T", x.Interface())
   426  	}
   427  }
   428  
   429  func TestParseErrors(t *testing.T) {
   430  	filename := starlarktest.DataFile("syntax", "testdata/errors.star")
   431  	for _, chunk := range chunkedfile.Read(filename, t) {
   432  		_, err := syntax.Parse(filename, chunk.Source, 0)
   433  		switch err := err.(type) {
   434  		case nil:
   435  			// ok
   436  		case syntax.Error:
   437  			chunk.GotError(int(err.Pos.Line), err.Msg)
   438  		default:
   439  			t.Error(err)
   440  		}
   441  		chunk.Done()
   442  	}
   443  }
   444  
   445  func TestFilePortion(t *testing.T) {
   446  	// Imagine that the Starlark file or expression print(x.f) is extracted
   447  	// from the middle of a file in some hypothetical template language;
   448  	// see https://github.com/google/starlark-go/issues/346. For example:
   449  	// --
   450  	// {{loop x seq}}
   451  	//   {{print(x.f)}}
   452  	// {{end}}
   453  	// --
   454  	fp := syntax.FilePortion{Content: []byte("print(x.f)"), FirstLine: 2, FirstCol: 4}
   455  	file, err := syntax.Parse("foo.template", fp, 0)
   456  	if err != nil {
   457  		t.Fatal(err)
   458  	}
   459  	span := fmt.Sprint(file.Stmts[0].Span())
   460  	want := "foo.template:2:4 foo.template:2:14"
   461  	if span != want {
   462  		t.Errorf("wrong span: got %q, want %q", span, want)
   463  	}
   464  }
   465  
   466  // dataFile is the same as starlarktest.DataFile.
   467  // We make a copy to avoid a dependency cycle.
   468  var dataFile = func(pkgdir, filename string) string {
   469  	return filepath.Join(build.Default.GOPATH, "src/go.starlark.net", pkgdir, filename)
   470  }
   471  
   472  func BenchmarkParse(b *testing.B) {
   473  	filename := dataFile("syntax", "testdata/scan.star")
   474  	b.StopTimer()
   475  	data, err := ioutil.ReadFile(filename)
   476  	if err != nil {
   477  		b.Fatal(err)
   478  	}
   479  	b.StartTimer()
   480  
   481  	for i := 0; i < b.N; i++ {
   482  		_, err := syntax.Parse(filename, data, 0)
   483  		if err != nil {
   484  			b.Fatal(err)
   485  		}
   486  	}
   487  }
   488  

View as plain text