encode_test.go

Documentation: github.com/bytedance/sonic/ast

     1  /*
     2   * Copyright 2021 ByteDance Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package ast
    18  
    19  import (
    20      `encoding/json`
    21      `runtime`
    22      `sync`
    23      `testing`
    24      `strings`
    25  
    26      `github.com/bytedance/sonic/internal/native/types`
    27      `github.com/stretchr/testify/assert`
    28      `github.com/stretchr/testify/require`
    29  )
    30  
    31  func TestGC_Encode(t *testing.T) {
    32      if debugSyncGC {
    33          return
    34      }
    35      root, err := NewSearcher(_TwitterJson).GetByPath()
    36      if err != nil {
    37          t.Fatal(err)
    38      }
    39      root.LoadAll()
    40      _, err = root.MarshalJSON()
    41      if err != nil {
    42          t.Fatal(err)
    43      }
    44      wg := &sync.WaitGroup{}
    45      N := 10000
    46      for i:=0; i<N; i++ {
    47          wg.Add(1)
    48          go func (wg *sync.WaitGroup)  {
    49              defer wg.Done()
    50              root, err := NewSearcher(_TwitterJson).GetByPath()
    51              if err != nil {
    52                  t.Error(err)
    53                  return
    54              }
    55              root.Load()
    56              _, err = root.MarshalJSON()
    57              if err != nil {
    58                  t.Error(err)
    59                  return
    60              }
    61              runtime.GC()
    62          }(wg)
    63      }
    64      wg.Wait()
    65  }
    66  
    67  func TestEncodeValue(t *testing.T) {
    68      obj := new(_TwitterStruct)
    69      if err := json.Unmarshal([]byte(_TwitterJson), obj); err != nil {
    70          t.Fatal(err)
    71      }
    72      // buf, err := encoder.Encode(obj, encoder.EscapeHTML|encoder.SortMapKeys)
    73      buf, err := json.Marshal(obj)
    74      if err != nil {
    75          t.Fatal(err)
    76      }
    77      quote, err := json.Marshal(_TwitterJson)
    78      if err != nil {
    79          t.Fatal(err)
    80      }
    81      type Case struct {
    82          node Node
    83          exp string
    84          err bool
    85      }
    86      input := []Case{
    87          {NewNull(), "null", false},
    88          {NewBool(true), "true", false},
    89          {NewBool(false), "false", false},
    90          {NewNumber("0.0"), "0.0", false},
    91          {NewString(""), `""`, false},
    92          {NewString(`\"\"`), `"\\\"\\\""`, false},
    93          {NewString(_TwitterJson), string(quote), false},
    94          {NewArray([]Node{}), "[]", false},
    95          {NewArray([]Node{NewString(""), NewNull()}), `["",null]`, false},
    96          {NewArray([]Node{NewBool(true), NewString("true"), NewString("\t")}), `[true,"true","\t"]`, false},
    97          {NewObject([]Pair{Pair{"a", NewNull()}, Pair{"b", NewNumber("0")}}), `{"a":null,"b":0}`, false},
    98          {NewObject([]Pair{Pair{"\ta", NewString("\t")}, Pair{"\bb", NewString("\b")}, Pair{"\nb", NewString("\n")}, Pair{"\ra", NewString("\r")}}),`{"\ta":"\t","\u0008b":"\u0008","\nb":"\n","\ra":"\r"}`, false},
    99          {NewObject([]Pair{}), `{}`, false},
   100          {NewObject([]Pair{Pair{Key: "", Value: NewNull()}}), `{"":null}`, false},
   101          {NewBytes([]byte("hello, world")), `"aGVsbG8sIHdvcmxk"`, false},
   102          {NewAny(obj), string(buf), false},
   103          {NewRaw(`[{ }]`), "[{}]", false},
   104          {Node{}, "", true},
   105          {Node{t: types.ValueType(1)}, "", true},
   106      }
   107      for i, c := range input {
   108          t.Log(i)
   109          buf, err := json.Marshal(&c.node)
   110          if c.err {
   111              if err == nil {
   112                  t.Fatal(i)
   113              }
   114              continue
   115          }
   116          if err != nil {
   117              t.Fatal(i, err)
   118          }
   119          assert.Equal(t, c.exp, string(buf))
   120      }
   121  }
   122  
   123  func TestEncodeNode(t *testing.T) {
   124      data := `{"a":[{},[],-0.1,true,false,null,""],"b":0,"c":true,"d":false,"e":null,"g":""}`
   125      root, e := NewSearcher(data).GetByPath()
   126      if e != nil {
   127          t.Fatal(root)
   128      }
   129      ret, err := root.MarshalJSON()
   130      if err != nil {
   131          t.Fatal(err)
   132      }
   133      if string(ret) != data {
   134          t.Fatal(string(ret))
   135      }
   136      root.skipAllKey()
   137      ret, err = root.MarshalJSON()
   138      if err != nil {
   139          t.Fatal(err)
   140      }
   141      if string(ret) != data {
   142          t.Fatal(string(ret))
   143      }
   144      root.loadAllKey()
   145      ret, err = root.MarshalJSON()
   146      if err != nil {
   147          t.Fatal(err)
   148      }
   149      if string(ret) != data {
   150          t.Fatal(string(ret))
   151      }
   152  }
   153  
   154  type SortableNode struct {
   155      sorted bool
   156  	*Node
   157  }
   158  
   159  func (j *SortableNode) UnmarshalJSON(data []byte) (error) {
   160      j.Node = new(Node)
   161  	return j.Node.UnmarshalJSON(data)
   162  }
   163  
   164  func (j *SortableNode) MarshalJSON() ([]byte, error) {
   165      if !j.sorted {
   166          j.Node.SortKeys(true)
   167          j.sorted = true
   168      }
   169  	return j.Node.MarshalJSON()
   170  }
   171  
   172  func TestMarshalSort(t *testing.T) {
   173      var data = `{"d":3,"a":{"c":1,"b":2},"e":null}`
   174      var obj map[string]*SortableNode
   175      require.NoError(t, json.Unmarshal([]byte(data), &obj))
   176      out, err := json.Marshal(obj)
   177      require.NoError(t, err)
   178      require.Equal(t, `{"a":{"b":2,"c":1},"d":3,"e":null}`, string(out))
   179      out, err = json.Marshal(obj)
   180      require.NoError(t, err)
   181      require.Equal(t, `{"a":{"b":2,"c":1},"d":3,"e":null}`, string(out))
   182  }
   183  
   184  func BenchmarkEncodeRaw_Sonic(b *testing.B) {
   185      data := _TwitterJson
   186      root, e := NewSearcher(data).GetByPath()
   187      if e != nil {
   188          b.Fatal(root)
   189      }
   190      _, err := root.MarshalJSON()
   191      if err != nil {
   192          b.Fatal(err)
   193      }
   194      b.SetBytes(int64(len(data)))
   195      b.ResetTimer()
   196      for i:=0; i<b.N; i++ {
   197          _, err := root.MarshalJSON()
   198          if err != nil {
   199              b.Fatal(err)
   200          }
   201      }
   202  }
   203  
   204  func BenchmarkEncodeSkip_Sonic(b *testing.B) {
   205      data := _TwitterJson
   206      root, e := NewParser(data).Parse()
   207      if e != 0 {
   208          b.Fatal(root)
   209      }
   210      root.skipAllKey()
   211      _, err := root.MarshalJSON()
   212      if err != nil {
   213          b.Fatal(err)
   214      }
   215      b.SetBytes(int64(len(data)))
   216      b.ResetTimer()
   217      for i:=0; i<b.N; i++ {
   218          _, err := root.MarshalJSON()
   219          if err != nil {
   220              b.Fatal(err)
   221          }
   222      }
   223  }
   224  
   225  func BenchmarkEncodeLoad_Sonic(b *testing.B) {
   226      data := _TwitterJson
   227      root, e := NewParser(data).Parse()
   228      if e != 0 {
   229          b.Fatal(root)
   230      }
   231      root.loadAllKey()
   232      _, err := root.MarshalJSON()
   233      if err != nil {
   234          b.Fatal(err)
   235      }
   236      b.SetBytes(int64(len(data)))
   237      b.ResetTimer()
   238      for i:=0; i<b.N; i++ {
   239          _, err := root.MarshalJSON()
   240          if err != nil {
   241              b.Fatal(err)
   242          }
   243      }
   244  }
   245  
   246  func TestEncodeNone(t *testing.T) {
   247      n := NewObject([]Pair{{Key:"a", Value:Node{}}})
   248      out, err := n.MarshalJSON()
   249      require.NoError(t, err)
   250      require.Equal(t, "{}", string(out))
   251      n = NewObject([]Pair{{Key:"a", Value:NewNull()}, {Key:"b", Value:Node{}}})
   252      out, err = n.MarshalJSON()
   253      require.NoError(t, err)
   254      require.Equal(t, `{"a":null}`, string(out))
   255  
   256      n = NewArray([]Node{Node{}})
   257      out, err = n.MarshalJSON()
   258      require.NoError(t, err)
   259      require.Equal(t, "[]", string(out))
   260      n = NewArray([]Node{NewNull(), Node{}})
   261      out, err = n.MarshalJSON()
   262      require.NoError(t, err)
   263      require.Equal(t, `[null]`, string(out))
   264  }
   265  
   266  
   267  type Path = []interface{}
   268  
   269  type testGetApi struct {
   270      json      string
   271      path      Path
   272  }
   273  
   274  type checkError func(error) bool
   275  
   276  func isSyntaxError(err error) bool {
   277      if err == nil {
   278          return false
   279      }
   280      return strings.HasPrefix(err.Error(), `"Syntax error at index`)
   281  }
   282  
   283  func isEmptySource(err error) bool {
   284      if err == nil {
   285          return false
   286      }
   287      return strings.Contains(err.Error(), "no sources available")
   288  }
   289  
   290  func isErrNotExist(err error) bool {
   291      return err == ErrNotExist
   292  }
   293  
   294  func isErrUnsupportType(err error) bool {
   295      return err == ErrUnsupportType
   296  }
   297  
   298  func testSyntaxJson(t *testing.T, json string, path ...interface{}) {
   299      search := NewSearcher(json)
   300      _, err := search.GetByPath(path...)
   301      assert.True(t, isSyntaxError(err))
   302  }
   303  
   304  func TestGetFromEmptyJson(t *testing.T) {
   305      tests := []testGetApi {
   306          { "", nil },
   307          { "", Path{}},
   308          { "", Path{""}},
   309          { "", Path{0}},
   310          { "", Path{"", ""}},
   311      }
   312      for _, test := range tests {
   313          f := func(t *testing.T) {
   314              search := NewSearcher(test.json)
   315              _, err := search.GetByPath(test.path...)
   316              assert.True(t, isEmptySource(err))
   317          }
   318          t.Run(test.json, f)
   319      }
   320  }
   321  
   322  func TestGetFromSyntaxError(t *testing.T) {
   323      tests := []testGetApi {
   324          { " \r\n\f\t", Path{} },
   325          { "123.", Path{} },
   326          { "+124", Path{} },
   327          { "-", Path{} },
   328          { "-e123", Path{} },
   329          { "-1.e123", Path{} },
   330          { "-12e456.1", Path{} },
   331          { "-12e.1", Path{} },
   332          { "[", Path{} },
   333          { "{", Path{} },
   334          { "[}", Path{} },
   335          { "{]", Path{} },
   336          { "{,}", Path{} },
   337          { "[,]", Path{} },
   338          { "tru", Path{} },
   339          { "fals", Path{} },
   340          { "nul", Path{} },
   341          { `{"a":"`, Path{"a"} },
   342          { `{"`, Path{} },
   343          { `"`, Path{} },
   344          { `"\"`, Path{} },
   345          { `"\\\"`, Path{} },
   346          { `"hello`, Path{} },
   347          { `{{}}`, Path{} },
   348          { `{[]}`, Path{} },
   349          { `{:,}`, Path{} },
   350          { `{test:error}`, Path{} },
   351          { `{":true}`, Path{} },
   352          { `{"" false}`, Path{} },
   353          { `{ "" : "false }`, Path{} },
   354          { `{"":"",}`, Path{} },
   355          { `{ " test : true}`, Path{} },
   356          { `{ "test" : tru }`, Path{} },
   357          { `{ "test" : true , }`, Path{} },
   358          { `{ {"test" : true , } }`, Path{} },
   359          { `{"test":1. }`, Path{} },
   360          { `{"\\\""`, Path{} },
   361          { `{"\\\"":`, Path{} },
   362          { `{"\\\":",""}`, Path{} },
   363          { `[{]`, Path{} },
   364          { `[tru]`, Path{} },
   365          { `[-1.]`, Path{} },
   366          { `[[]`, Path{} },
   367          { `[[],`, Path{} },
   368          { `[ true , false , [ ]`, Path{} },
   369          { `[true, false, [],`, Path{} },
   370          { `[true, false, [],]`, Path{} },
   371          { `{"key": [true, false, []], "key2": {{}}`, Path{} },
   372      }
   373  
   374      for _, test := range tests {
   375          f := func(t *testing.T) {
   376              testSyntaxJson(t, test.json, test.path...)
   377              path := append(Path{"key"}, test.path...)
   378              testSyntaxJson(t, `{"key":` + test.json, path...)
   379              path  = append(Path{""}, test.path...)
   380              testSyntaxJson(t, `{"":` + test.json, path...)
   381              path  = append(Path{1}, test.path...)
   382              testSyntaxJson(t, `["",` + test.json, path...)
   383          }
   384          t.Run(test.json, f)
   385      }
   386  }
   387  
   388  // NOTE: GetByPath API not validate the undemanded fields for performance.
   389  func TestGetWithInvalidUndemandedField(t *testing.T) {
   390      type Any = interface{}
   391      tests := []struct {
   392          json string
   393          path Path
   394          exp  Any
   395      } {
   396          { "-0xyz", Path{}, Any(float64(-0))},
   397          { "-12e4xyz", Path{}, Any(float64(-12e4))},
   398          { "truex",  Path{}, Any(true)},
   399          { "false,", Path{}, Any(false)},
   400          { `{"a":{,xxx},"b":true}`, Path{"b"}, Any(true)},
   401          { `{"a":[,xxx],"b":true}`, Path{"b"}, Any(true)},
   402      }
   403  
   404      for _, test := range tests {
   405          f := func(t *testing.T) {
   406              search := NewSearcher(test.json)
   407              node, err := search.GetByPath(test.path...)
   408              assert.NoError(t, err)
   409              v, err := node.Interface()
   410              assert.NoError(t, err)
   411              assert.Equal(t, v, test.exp)
   412          }
   413          t.Run(test.json, f)
   414      }
   415  }
   416  
   417  func TestGet_InvalidPathType(t *testing.T) {
   418      assert.Panics(t, assert.PanicTestFunc(func() {
   419          data := `{"a":[{"b":true}]}`
   420          s := NewSearcher(data)
   421          s.GetByPath("a", true)
   422  
   423          s = NewSearcher(data)
   424          s.GetByPath("a", nil)
   425  
   426          s = NewSearcher(data)
   427          s.GetByPath("a", -1)
   428      }))
   429  }
   430
View as plain text