Source file src/github.com/mailru/easyjson/jlexer/lexer_test.go

Documentation: github.com/mailru/easyjson/jlexer

     1  package jlexer
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/json"
     6  	"reflect"
     7  	"testing"
     8  )
     9  
    10  func TestString(t *testing.T) {
    11  	for i, test := range []struct {
    12  		toParse   string
    13  		want      string
    14  		wantError bool
    15  	}{
    16  		{toParse: `"simple string"`, want: "simple string"},
    17  		{toParse: " \r\r\n\t  " + `"test"`, want: "test"},
    18  		{toParse: `"\n\t\"\/\\\f\r"`, want: "\n\t\"/\\\f\r"},
    19  		{toParse: `"\u0020"`, want: " "},
    20  		{toParse: `"\u0020-\t"`, want: " -\t"},
    21  		{toParse: `"\ufffd\uFFFD"`, want: "\ufffd\ufffd"},
    22  		{toParse: `"\ud83d\ude00"`, want: "😀"},
    23  		{toParse: `"\ud83d\ude08"`, want: "😈"},
    24  		{toParse: `"\ud8"`, wantError: true},
    25  
    26  		{toParse: `"test"junk`, want: "test"},
    27  
    28  		{toParse: `5`, wantError: true},    // not a string
    29  		{toParse: `"\x"`, wantError: true}, // invalid escape
    30  		{toParse: `"\ud800"`, want: "�"},   // invalid utf-8 char; return replacement char
    31  	} {
    32  		{
    33  			l := Lexer{Data: []byte(test.toParse)}
    34  
    35  			got := l.String()
    36  			if got != test.want {
    37  				t.Errorf("[%d, %q] String() = %v; want %v", i, test.toParse, got, test.want)
    38  			}
    39  			err := l.Error()
    40  			if err != nil && !test.wantError {
    41  				t.Errorf("[%d, %q] String() error: %v", i, test.toParse, err)
    42  			} else if err == nil && test.wantError {
    43  				t.Errorf("[%d, %q] String() ok; want error", i, test.toParse)
    44  			}
    45  		}
    46  		{
    47  			l := Lexer{Data: []byte(test.toParse)}
    48  
    49  			got := l.StringIntern()
    50  			if got != test.want {
    51  				t.Errorf("[%d, %q] String() = %v; want %v", i, test.toParse, got, test.want)
    52  			}
    53  			err := l.Error()
    54  			if err != nil && !test.wantError {
    55  				t.Errorf("[%d, %q] String() error: %v", i, test.toParse, err)
    56  			} else if err == nil && test.wantError {
    57  				t.Errorf("[%d, %q] String() ok; want error", i, test.toParse)
    58  			}
    59  		}
    60  	}
    61  }
    62  
    63  func TestStringIntern(t *testing.T) {
    64  	data := []byte(`"string interning test"`)
    65  	var l Lexer
    66  
    67  	allocsPerRun := testing.AllocsPerRun(1000, func() {
    68  		l = Lexer{Data: data}
    69  		_ = l.StringIntern()
    70  	})
    71  	if allocsPerRun != 0 {
    72  		t.Fatalf("expected 0 allocs, got %f", allocsPerRun)
    73  	}
    74  
    75  	allocsPerRun = testing.AllocsPerRun(1000, func() {
    76  		l = Lexer{Data: data}
    77  		_ = l.String()
    78  	})
    79  	if allocsPerRun != 1 {
    80  		t.Fatalf("expected 1 allocs, got %f", allocsPerRun)
    81  	}
    82  }
    83  
    84  func TestBytes(t *testing.T) {
    85  	for i, test := range []struct {
    86  		toParse   string
    87  		want      string
    88  		wantError bool
    89  	}{
    90  		{toParse: `"c2ltcGxlIHN0cmluZw=="`, want: "simple string"},
    91  		{toParse: " \r\r\n\t  " + `"dGVzdA=="`, want: "test"},
    92  		{toParse: `"c3ViamVjdHM\/X2Q9MQ=="`, want: "subjects?_d=1"}, // base64 with forward slash escaped
    93  
    94  		{toParse: `5`, wantError: true},                     // not a JSON string
    95  		{toParse: `"foobar"`, wantError: true},              // not base64 encoded
    96  		{toParse: `"c2ltcGxlIHN0cmluZw="`, wantError: true}, // invalid base64 padding
    97  	} {
    98  		l := Lexer{Data: []byte(test.toParse)}
    99  
   100  		got := l.Bytes()
   101  		if bytes.Compare(got, []byte(test.want)) != 0 {
   102  			t.Errorf("[%d, %q] Bytes() = %v; want: %v", i, test.toParse, got, []byte(test.want))
   103  		}
   104  		err := l.Error()
   105  		if err != nil && !test.wantError {
   106  			t.Errorf("[%d, %q] Bytes() error: %v", i, test.toParse, err)
   107  		} else if err == nil && test.wantError {
   108  			t.Errorf("[%d, %q] Bytes() ok; want error", i, test.toParse)
   109  		}
   110  	}
   111  }
   112  
   113  func TestNumber(t *testing.T) {
   114  	for i, test := range []struct {
   115  		toParse   string
   116  		want      string
   117  		wantError bool
   118  	}{
   119  		{toParse: "123", want: "123"},
   120  		{toParse: "-123", want: "-123"},
   121  		{toParse: "\r\n12.35", want: "12.35"},
   122  		{toParse: "12.35e+1", want: "12.35e+1"},
   123  		{toParse: "12.35e-15", want: "12.35e-15"},
   124  		{toParse: "12.35E-15", want: "12.35E-15"},
   125  		{toParse: "12.35E15", want: "12.35E15"},
   126  
   127  		{toParse: `"a"`, wantError: true},
   128  		{toParse: "123junk", wantError: true},
   129  		{toParse: "1.2.3", wantError: true},
   130  		{toParse: "1e2e3", wantError: true},
   131  		{toParse: "1e2.3", wantError: true},
   132  	} {
   133  		l := Lexer{Data: []byte(test.toParse)}
   134  
   135  		got := l.number()
   136  		if got != test.want {
   137  			t.Errorf("[%d, %q] number() = %v; want %v", i, test.toParse, got, test.want)
   138  		}
   139  		err := l.Error()
   140  		if err != nil && !test.wantError {
   141  			t.Errorf("[%d, %q] number() error: %v", i, test.toParse, err)
   142  		} else if err == nil && test.wantError {
   143  			t.Errorf("[%d, %q] number() ok; want error", i, test.toParse)
   144  		}
   145  	}
   146  }
   147  
   148  func TestBool(t *testing.T) {
   149  	for i, test := range []struct {
   150  		toParse   string
   151  		want      bool
   152  		wantError bool
   153  	}{
   154  		{toParse: "true", want: true},
   155  		{toParse: "false", want: false},
   156  
   157  		{toParse: "1", wantError: true},
   158  		{toParse: "truejunk", wantError: true},
   159  		{toParse: `false"junk"`, wantError: true},
   160  		{toParse: "True", wantError: true},
   161  		{toParse: "False", wantError: true},
   162  	} {
   163  		l := Lexer{Data: []byte(test.toParse)}
   164  
   165  		got := l.Bool()
   166  		if got != test.want {
   167  			t.Errorf("[%d, %q] Bool() = %v; want %v", i, test.toParse, got, test.want)
   168  		}
   169  		err := l.Error()
   170  		if err != nil && !test.wantError {
   171  			t.Errorf("[%d, %q] Bool() error: %v", i, test.toParse, err)
   172  		} else if err == nil && test.wantError {
   173  			t.Errorf("[%d, %q] Bool() ok; want error", i, test.toParse)
   174  		}
   175  	}
   176  }
   177  
   178  func TestSkipRecursive(t *testing.T) {
   179  	for i, test := range []struct {
   180  		toParse   string
   181  		left      string
   182  		wantError bool
   183  	}{
   184  		{toParse: "5, 4", left: ", 4"},
   185  		{toParse: "[5, 6], 4", left: ", 4"},
   186  		{toParse: "[5, [7,8]]: 4", left: ": 4"},
   187  
   188  		{toParse: `{"a":1}, 4`, left: ", 4"},
   189  		{toParse: `{"a":1, "b":{"c": 5}, "e":[12,15]}, 4`, left: ", 4"},
   190  
   191  		// array start/end chars in a string
   192  		{toParse: `[5, "]"], 4`, left: ", 4"},
   193  		{toParse: `[5, "\"]"], 4`, left: ", 4"},
   194  		{toParse: `[5, "["], 4`, left: ", 4"},
   195  		{toParse: `[5, "\"["], 4`, left: ", 4"},
   196  
   197  		// object start/end chars in a string
   198  		{toParse: `{"a}":1}, 4`, left: ", 4"},
   199  		{toParse: `{"a\"}":1}, 4`, left: ", 4"},
   200  		{toParse: `{"a{":1}, 4`, left: ", 4"},
   201  		{toParse: `{"a\"{":1}, 4`, left: ", 4"},
   202  
   203  		// object with double slashes at the end of string
   204  		{toParse: `{"a":"hey\\"}, 4`, left: ", 4"},
   205  
   206  		// make sure skipping an invalid json results in an error
   207  		{toParse: `{"a": [ ##invalid json## ]}, 4`, wantError: true},
   208  		{toParse: `{"a": [ [1], [ ##invalid json## ]]}, 4`, wantError: true},
   209  	} {
   210  		l := Lexer{Data: []byte(test.toParse)}
   211  
   212  		l.SkipRecursive()
   213  
   214  		got := string(l.Data[l.pos:])
   215  		if got != test.left {
   216  			t.Errorf("[%d, %q] SkipRecursive() left = %v; want %v", i, test.toParse, got, test.left)
   217  		}
   218  		err := l.Error()
   219  		if err != nil && !test.wantError {
   220  			t.Errorf("[%d, %q] SkipRecursive() error: %v", i, test.toParse, err)
   221  		} else if err == nil && test.wantError {
   222  			t.Errorf("[%d, %q] SkipRecursive() ok; want error", i, test.toParse)
   223  		}
   224  	}
   225  }
   226  
   227  func TestInterface(t *testing.T) {
   228  	for i, test := range []struct {
   229  		toParse   string
   230  		want      interface{}
   231  		wantError bool
   232  	}{
   233  		{toParse: "null", want: nil},
   234  		{toParse: "true", want: true},
   235  		{toParse: `"a"`, want: "a"},
   236  		{toParse: "5", want: float64(5)},
   237  
   238  		{toParse: `{}`, want: map[string]interface{}{}},
   239  		{toParse: `[]`, want: []interface{}{}},
   240  
   241  		{toParse: `{"a": "b"}`, want: map[string]interface{}{"a": "b"}},
   242  		{toParse: `[5]`, want: []interface{}{float64(5)}},
   243  
   244  		{toParse: `{"a":5 , "b" : "string"}`, want: map[string]interface{}{"a": float64(5), "b": "string"}},
   245  		{toParse: `["a", 5 , null, true]`, want: []interface{}{"a", float64(5), nil, true}},
   246  
   247  		{toParse: `{"a" "b"}`, wantError: true},
   248  		{toParse: `{"a": "b",}`, wantError: true},
   249  		{toParse: `{"a":"b","c" "b"}`, wantError: true},
   250  		{toParse: `{"a": "b","c":"d",}`, wantError: true},
   251  		{toParse: `{,}`, wantError: true},
   252  
   253  		{toParse: `[1, 2,]`, wantError: true},
   254  		{toParse: `[1  2]`, wantError: true},
   255  		{toParse: `[,]`, wantError: true},
   256  	} {
   257  		l := Lexer{Data: []byte(test.toParse)}
   258  
   259  		got := l.Interface()
   260  		if !reflect.DeepEqual(got, test.want) {
   261  			t.Errorf("[%d, %q] Interface() = %v; want %v", i, test.toParse, got, test.want)
   262  		}
   263  		err := l.Error()
   264  		if err != nil && !test.wantError {
   265  			t.Errorf("[%d, %q] Interface() error: %v", i, test.toParse, err)
   266  		} else if err == nil && test.wantError {
   267  			t.Errorf("[%d, %q] Interface() ok; want error", i, test.toParse)
   268  		}
   269  	}
   270  }
   271  
   272  func TestConsumed(t *testing.T) {
   273  	for i, test := range []struct {
   274  		toParse   string
   275  		wantError bool
   276  	}{
   277  		{toParse: "", wantError: false},
   278  		{toParse: "   ", wantError: false},
   279  		{toParse: "\r\n", wantError: false},
   280  		{toParse: "\t\t", wantError: false},
   281  
   282  		{toParse: "{", wantError: true},
   283  	} {
   284  		l := Lexer{Data: []byte(test.toParse)}
   285  		l.Consumed()
   286  
   287  		err := l.Error()
   288  		if err != nil && !test.wantError {
   289  			t.Errorf("[%d, %q] Consumed() error: %v", i, test.toParse, err)
   290  		} else if err == nil && test.wantError {
   291  			t.Errorf("[%d, %q] Consumed() ok; want error", i, test.toParse)
   292  		}
   293  	}
   294  }
   295  
   296  func TestJsonNumber(t *testing.T) {
   297  	for i, test := range []struct {
   298  		toParse        string
   299  		want           json.Number
   300  		wantLexerError bool
   301  		wantValue      interface{}
   302  		wantValueError bool
   303  	}{
   304  		{toParse: `10`, want: json.Number("10"), wantValue: int64(10)},
   305  		{toParse: `0`, want: json.Number("0"), wantValue: int64(0)},
   306  		{toParse: `0.12`, want: json.Number("0.12"), wantValue: 0.12},
   307  		{toParse: `25E-4`, want: json.Number("25E-4"), wantValue: 25e-4},
   308  
   309  		{toParse: `"10"`, want: json.Number("10"), wantValue: int64(10)},
   310  		{toParse: `"0"`, want: json.Number("0"), wantValue: int64(0)},
   311  		{toParse: `"0.12"`, want: json.Number("0.12"), wantValue: 0.12},
   312  		{toParse: `"25E-4"`, want: json.Number("25E-4"), wantValue: 25e-4},
   313  
   314  		{toParse: `"foo"`, want: json.Number("foo"), wantValueError: true},
   315  		{toParse: `null`, want: json.Number(""), wantValueError: true},
   316  
   317  		{toParse: `"a""`, want: json.Number("a"), wantValueError: true},
   318  
   319  		{toParse: `[1]`, want: json.Number(""), wantLexerError: true, wantValueError: true},
   320  		{toParse: `{}`, want: json.Number(""), wantLexerError: true, wantValueError: true},
   321  		{toParse: `a`, want: json.Number(""), wantLexerError: true, wantValueError: true},
   322  	} {
   323  		l := Lexer{Data: []byte(test.toParse)}
   324  
   325  		got := l.JsonNumber()
   326  		if got != test.want {
   327  			t.Errorf("[%d, %q] JsonNumber() = %v; want %v", i, test.toParse, got, test.want)
   328  		}
   329  
   330  		err := l.Error()
   331  		if err != nil && !test.wantLexerError {
   332  			t.Errorf("[%d, %q] JsonNumber() lexer error: %v", i, test.toParse, err)
   333  		} else if err == nil && test.wantLexerError {
   334  			t.Errorf("[%d, %q] JsonNumber() ok; want lexer error", i, test.toParse)
   335  		}
   336  
   337  		var valueErr error
   338  		var gotValue interface{}
   339  		switch test.wantValue.(type) {
   340  		case float64:
   341  			gotValue, valueErr = got.Float64()
   342  		default:
   343  			gotValue, valueErr = got.Int64()
   344  		}
   345  
   346  		if !reflect.DeepEqual(gotValue, test.wantValue) && !test.wantLexerError && !test.wantValueError {
   347  			t.Errorf("[%d, %q] JsonNumber() = %v; want %v", i, test.toParse, gotValue, test.wantValue)
   348  		}
   349  
   350  		if valueErr != nil && !test.wantValueError {
   351  			t.Errorf("[%d, %q] JsonNumber() value error: %v", i, test.toParse, valueErr)
   352  		} else if valueErr == nil && test.wantValueError {
   353  			t.Errorf("[%d, %q] JsonNumber() ok; want value error", i, test.toParse)
   354  		}
   355  	}
   356  }
   357  
   358  func TestFetchStringUnterminatedString(t *testing.T) {
   359  	for _, test := range []struct {
   360  		data []byte
   361  	}{
   362  		{data: []byte(`"sting without trailing quote`)},
   363  		{data: []byte(`"\"`)},
   364  		{data: []byte{'"'}},
   365  	} {
   366  		l := Lexer{Data: test.data}
   367  		l.fetchString()
   368  		if l.pos > len(l.Data) {
   369  			t.Errorf("fetchString(%s): pos=%v should not be greater than length of Data = %v", test.data, l.pos, len(l.Data))
   370  		}
   371  		if l.Error() == nil {
   372  			t.Errorf("fetchString(%s): should add parsing error", test.data)
   373  		}
   374  	}
   375  }
   376  

View as plain text