token.go

Documentation: k8s.io/kube-openapi/pkg/internal/third_party/go-json-experiment/json

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package json
     6  
     7  import (
     8  	"math"
     9  	"strconv"
    10  )
    11  
    12  // NOTE: Token is analogous to v1 json.Token.
    13  
    14  const (
    15  	maxInt64  = math.MaxInt64
    16  	minInt64  = math.MinInt64
    17  	maxUint64 = math.MaxUint64
    18  	minUint64 = 0 // for consistency and readability purposes
    19  
    20  	invalidTokenPanic = "invalid json.Token; it has been voided by a subsequent json.Decoder call"
    21  )
    22  
    23  // Token represents a lexical JSON token, which may be one of the following:
    24  //   - a JSON literal (i.e., null, true, or false)
    25  //   - a JSON string (e.g., "hello, world!")
    26  //   - a JSON number (e.g., 123.456)
    27  //   - a start or end delimiter for a JSON object (i.e., { or } )
    28  //   - a start or end delimiter for a JSON array (i.e., [ or ] )
    29  //
    30  // A Token cannot represent entire array or object values, while a RawValue can.
    31  // There is no Token to represent commas and colons since
    32  // these structural tokens can be inferred from the surrounding context.
    33  type Token struct {
    34  	nonComparable
    35  
    36  	// Tokens can exist in either a "raw" or an "exact" form.
    37  	// Tokens produced by the Decoder are in the "raw" form.
    38  	// Tokens returned by constructors are usually in the "exact" form.
    39  	// The Encoder accepts Tokens in either the "raw" or "exact" form.
    40  	//
    41  	// The following chart shows the possible values for each Token type:
    42  	//	╔═════════════════╦════════════╤════════════╤════════════╗
    43  	//	║ Token type      ║ raw field  │ str field  │ num field  ║
    44  	//	╠═════════════════╬════════════╪════════════╪════════════╣
    45  	//	║ null   (raw)    ║ "null"     │ ""         │ 0          ║
    46  	//	║ false  (raw)    ║ "false"    │ ""         │ 0          ║
    47  	//	║ true   (raw)    ║ "true"     │ ""         │ 0          ║
    48  	//	║ string (raw)    ║ non-empty  │ ""         │ offset     ║
    49  	//	║ string (string) ║ nil        │ non-empty  │ 0          ║
    50  	//	║ number (raw)    ║ non-empty  │ ""         │ offset     ║
    51  	//	║ number (float)  ║ nil        │ "f"        │ non-zero   ║
    52  	//	║ number (int64)  ║ nil        │ "i"        │ non-zero   ║
    53  	//	║ number (uint64) ║ nil        │ "u"        │ non-zero   ║
    54  	//	║ object (delim)  ║ "{" or "}" │ ""         │ 0          ║
    55  	//	║ array  (delim)  ║ "[" or "]" │ ""         │ 0          ║
    56  	//	╚═════════════════╩════════════╧════════════╧════════════╝
    57  	//
    58  	// Notes:
    59  	//   - For tokens stored in "raw" form, the num field contains the
    60  	//     absolute offset determined by raw.previousOffsetStart().
    61  	//     The buffer itself is stored in raw.previousBuffer().
    62  	//   - JSON literals and structural characters are always in the "raw" form.
    63  	//   - JSON strings and numbers can be in either "raw" or "exact" forms.
    64  	//   - The exact zero value of JSON strings and numbers in the "exact" forms
    65  	//     have ambiguous representation. Thus, they are always represented
    66  	//     in the "raw" form.
    67  
    68  	// raw contains a reference to the raw decode buffer.
    69  	// If non-nil, then its value takes precedence over str and num.
    70  	// It is only valid if num == raw.previousOffsetStart().
    71  	raw *decodeBuffer
    72  
    73  	// str is the unescaped JSON string if num is zero.
    74  	// Otherwise, it is "f", "i", or "u" if num should be interpreted
    75  	// as a float64, int64, or uint64, respectively.
    76  	str string
    77  
    78  	// num is a float64, int64, or uint64 stored as a uint64 value.
    79  	// It is non-zero for any JSON number in the "exact" form.
    80  	num uint64
    81  }
    82  
    83  // TODO: Does representing 1-byte delimiters as *decodeBuffer cause performance issues?
    84  
    85  var (
    86  	Null  Token = rawToken("null")
    87  	False Token = rawToken("false")
    88  	True  Token = rawToken("true")
    89  
    90  	ObjectStart Token = rawToken("{")
    91  	ObjectEnd   Token = rawToken("}")
    92  	ArrayStart  Token = rawToken("[")
    93  	ArrayEnd    Token = rawToken("]")
    94  
    95  	zeroString Token = rawToken(`""`)
    96  	zeroNumber Token = rawToken(`0`)
    97  
    98  	nanString  Token = String("NaN")
    99  	pinfString Token = String("Infinity")
   100  	ninfString Token = String("-Infinity")
   101  )
   102  
   103  func rawToken(s string) Token {
   104  	return Token{raw: &decodeBuffer{buf: []byte(s), prevStart: 0, prevEnd: len(s)}}
   105  }
   106  
   107  // Bool constructs a Token representing a JSON boolean.
   108  func Bool(b bool) Token {
   109  	if b {
   110  		return True
   111  	}
   112  	return False
   113  }
   114  
   115  // String constructs a Token representing a JSON string.
   116  // The provided string should contain valid UTF-8, otherwise invalid characters
   117  // may be mangled as the Unicode replacement character.
   118  func String(s string) Token {
   119  	if len(s) == 0 {
   120  		return zeroString
   121  	}
   122  	return Token{str: s}
   123  }
   124  
   125  // Float constructs a Token representing a JSON number.
   126  // The values NaN, +Inf, and -Inf will be represented
   127  // as a JSON string with the values "NaN", "Infinity", and "-Infinity".
   128  func Float(n float64) Token {
   129  	switch {
   130  	case math.Float64bits(n) == 0:
   131  		return zeroNumber
   132  	case math.IsNaN(n):
   133  		return nanString
   134  	case math.IsInf(n, +1):
   135  		return pinfString
   136  	case math.IsInf(n, -1):
   137  		return ninfString
   138  	}
   139  	return Token{str: "f", num: math.Float64bits(n)}
   140  }
   141  
   142  // Int constructs a Token representing a JSON number from an int64.
   143  func Int(n int64) Token {
   144  	if n == 0 {
   145  		return zeroNumber
   146  	}
   147  	return Token{str: "i", num: uint64(n)}
   148  }
   149  
   150  // Uint constructs a Token representing a JSON number from a uint64.
   151  func Uint(n uint64) Token {
   152  	if n == 0 {
   153  		return zeroNumber
   154  	}
   155  	return Token{str: "u", num: uint64(n)}
   156  }
   157  
   158  // Clone makes a copy of the Token such that its value remains valid
   159  // even after a subsequent Decoder.Read call.
   160  func (t Token) Clone() Token {
   161  	// TODO: Allow caller to avoid any allocations?
   162  	if raw := t.raw; raw != nil {
   163  		// Avoid copying globals.
   164  		if t.raw.prevStart == 0 {
   165  			switch t.raw {
   166  			case Null.raw:
   167  				return Null
   168  			case False.raw:
   169  				return False
   170  			case True.raw:
   171  				return True
   172  			case ObjectStart.raw:
   173  				return ObjectStart
   174  			case ObjectEnd.raw:
   175  				return ObjectEnd
   176  			case ArrayStart.raw:
   177  				return ArrayStart
   178  			case ArrayEnd.raw:
   179  				return ArrayEnd
   180  			}
   181  		}
   182  
   183  		if uint64(raw.previousOffsetStart()) != t.num {
   184  			panic(invalidTokenPanic)
   185  		}
   186  		// TODO(https://go.dev/issue/45038): Use bytes.Clone.
   187  		buf := append([]byte(nil), raw.previousBuffer()...)
   188  		return Token{raw: &decodeBuffer{buf: buf, prevStart: 0, prevEnd: len(buf)}}
   189  	}
   190  	return t
   191  }
   192  
   193  // Bool returns the value for a JSON boolean.
   194  // It panics if the token kind is not a JSON boolean.
   195  func (t Token) Bool() bool {
   196  	switch t.raw {
   197  	case True.raw:
   198  		return true
   199  	case False.raw:
   200  		return false
   201  	default:
   202  		panic("invalid JSON token kind: " + t.Kind().String())
   203  	}
   204  }
   205  
   206  // appendString appends a JSON string to dst and returns it.
   207  // It panics if t is not a JSON string.
   208  func (t Token) appendString(dst []byte, validateUTF8, preserveRaw bool, escapeRune func(rune) bool) ([]byte, error) {
   209  	if raw := t.raw; raw != nil {
   210  		// Handle raw string value.
   211  		buf := raw.previousBuffer()
   212  		if Kind(buf[0]) == '"' {
   213  			if escapeRune == nil && consumeSimpleString(buf) == len(buf) {
   214  				return append(dst, buf...), nil
   215  			}
   216  			dst, _, err := reformatString(dst, buf, validateUTF8, preserveRaw, escapeRune)
   217  			return dst, err
   218  		}
   219  	} else if len(t.str) != 0 && t.num == 0 {
   220  		// Handle exact string value.
   221  		return appendString(dst, t.str, validateUTF8, escapeRune)
   222  	}
   223  
   224  	panic("invalid JSON token kind: " + t.Kind().String())
   225  }
   226  
   227  // String returns the unescaped string value for a JSON string.
   228  // For other JSON kinds, this returns the raw JSON representation.
   229  func (t Token) String() string {
   230  	// This is inlinable to take advantage of "function outlining".
   231  	// This avoids an allocation for the string(b) conversion
   232  	// if the caller does not use the string in an escaping manner.
   233  	// See https://blog.filippo.io/efficient-go-apis-with-the-inliner/
   234  	s, b := t.string()
   235  	if len(b) > 0 {
   236  		return string(b)
   237  	}
   238  	return s
   239  }
   240  func (t Token) string() (string, []byte) {
   241  	if raw := t.raw; raw != nil {
   242  		if uint64(raw.previousOffsetStart()) != t.num {
   243  			panic(invalidTokenPanic)
   244  		}
   245  		buf := raw.previousBuffer()
   246  		if buf[0] == '"' {
   247  			// TODO: Preserve valueFlags in Token?
   248  			isVerbatim := consumeSimpleString(buf) == len(buf)
   249  			return "", unescapeStringMayCopy(buf, isVerbatim)
   250  		}
   251  		// Handle tokens that are not JSON strings for fmt.Stringer.
   252  		return "", buf
   253  	}
   254  	if len(t.str) != 0 && t.num == 0 {
   255  		return t.str, nil
   256  	}
   257  	// Handle tokens that are not JSON strings for fmt.Stringer.
   258  	if t.num > 0 {
   259  		switch t.str[0] {
   260  		case 'f':
   261  			return string(appendNumber(nil, math.Float64frombits(t.num), 64)), nil
   262  		case 'i':
   263  			return strconv.FormatInt(int64(t.num), 10), nil
   264  		case 'u':
   265  			return strconv.FormatUint(uint64(t.num), 10), nil
   266  		}
   267  	}
   268  	return "<invalid json.Token>", nil
   269  }
   270  
   271  // appendNumber appends a JSON number to dst and returns it.
   272  // It panics if t is not a JSON number.
   273  func (t Token) appendNumber(dst []byte, canonicalize bool) ([]byte, error) {
   274  	if raw := t.raw; raw != nil {
   275  		// Handle raw number value.
   276  		buf := raw.previousBuffer()
   277  		if Kind(buf[0]).normalize() == '0' {
   278  			if !canonicalize {
   279  				return append(dst, buf...), nil
   280  			}
   281  			dst, _, err := reformatNumber(dst, buf, canonicalize)
   282  			return dst, err
   283  		}
   284  	} else if t.num != 0 {
   285  		// Handle exact number value.
   286  		switch t.str[0] {
   287  		case 'f':
   288  			return appendNumber(dst, math.Float64frombits(t.num), 64), nil
   289  		case 'i':
   290  			return strconv.AppendInt(dst, int64(t.num), 10), nil
   291  		case 'u':
   292  			return strconv.AppendUint(dst, uint64(t.num), 10), nil
   293  		}
   294  	}
   295  
   296  	panic("invalid JSON token kind: " + t.Kind().String())
   297  }
   298  
   299  // Float returns the floating-point value for a JSON number.
   300  // It returns a NaN, +Inf, or -Inf value for any JSON string
   301  // with the values "NaN", "Infinity", or "-Infinity".
   302  // It panics for all other cases.
   303  func (t Token) Float() float64 {
   304  	if raw := t.raw; raw != nil {
   305  		// Handle raw number value.
   306  		if uint64(raw.previousOffsetStart()) != t.num {
   307  			panic(invalidTokenPanic)
   308  		}
   309  		buf := raw.previousBuffer()
   310  		if Kind(buf[0]).normalize() == '0' {
   311  			fv, _ := parseFloat(buf, 64)
   312  			return fv
   313  		}
   314  	} else if t.num != 0 {
   315  		// Handle exact number value.
   316  		switch t.str[0] {
   317  		case 'f':
   318  			return math.Float64frombits(t.num)
   319  		case 'i':
   320  			return float64(int64(t.num))
   321  		case 'u':
   322  			return float64(uint64(t.num))
   323  		}
   324  	}
   325  
   326  	// Handle string values with "NaN", "Infinity", or "-Infinity".
   327  	if t.Kind() == '"' {
   328  		switch t.String() {
   329  		case "NaN":
   330  			return math.NaN()
   331  		case "Infinity":
   332  			return math.Inf(+1)
   333  		case "-Infinity":
   334  			return math.Inf(-1)
   335  		}
   336  	}
   337  
   338  	panic("invalid JSON token kind: " + t.Kind().String())
   339  }
   340  
   341  // Int returns the signed integer value for a JSON number.
   342  // The fractional component of any number is ignored (truncation toward zero).
   343  // Any number beyond the representation of an int64 will be saturated
   344  // to the closest representable value.
   345  // It panics if the token kind is not a JSON number.
   346  func (t Token) Int() int64 {
   347  	if raw := t.raw; raw != nil {
   348  		// Handle raw integer value.
   349  		if uint64(raw.previousOffsetStart()) != t.num {
   350  			panic(invalidTokenPanic)
   351  		}
   352  		neg := false
   353  		buf := raw.previousBuffer()
   354  		if len(buf) > 0 && buf[0] == '-' {
   355  			neg, buf = true, buf[1:]
   356  		}
   357  		if numAbs, ok := parseDecUint(buf); ok {
   358  			if neg {
   359  				if numAbs > -minInt64 {
   360  					return minInt64
   361  				}
   362  				return -1 * int64(numAbs)
   363  			} else {
   364  				if numAbs > +maxInt64 {
   365  					return maxInt64
   366  				}
   367  				return +1 * int64(numAbs)
   368  			}
   369  		}
   370  	} else if t.num != 0 {
   371  		// Handle exact integer value.
   372  		switch t.str[0] {
   373  		case 'i':
   374  			return int64(t.num)
   375  		case 'u':
   376  			if t.num > maxInt64 {
   377  				return maxInt64
   378  			}
   379  			return int64(t.num)
   380  		}
   381  	}
   382  
   383  	// Handle JSON number that is a floating-point value.
   384  	if t.Kind() == '0' {
   385  		switch fv := t.Float(); {
   386  		case fv >= maxInt64:
   387  			return maxInt64
   388  		case fv <= minInt64:
   389  			return minInt64
   390  		default:
   391  			return int64(fv) // truncation toward zero
   392  		}
   393  	}
   394  
   395  	panic("invalid JSON token kind: " + t.Kind().String())
   396  }
   397  
   398  // Uint returns the unsigned integer value for a JSON number.
   399  // The fractional component of any number is ignored (truncation toward zero).
   400  // Any number beyond the representation of an uint64 will be saturated
   401  // to the closest representable value.
   402  // It panics if the token kind is not a JSON number.
   403  func (t Token) Uint() uint64 {
   404  	// NOTE: This accessor returns 0 for any negative JSON number,
   405  	// which might be surprising, but is at least consistent with the behavior
   406  	// of saturating out-of-bounds numbers to the closest representable number.
   407  
   408  	if raw := t.raw; raw != nil {
   409  		// Handle raw integer value.
   410  		if uint64(raw.previousOffsetStart()) != t.num {
   411  			panic(invalidTokenPanic)
   412  		}
   413  		neg := false
   414  		buf := raw.previousBuffer()
   415  		if len(buf) > 0 && buf[0] == '-' {
   416  			neg, buf = true, buf[1:]
   417  		}
   418  		if num, ok := parseDecUint(buf); ok {
   419  			if neg {
   420  				return minUint64
   421  			}
   422  			return num
   423  		}
   424  	} else if t.num != 0 {
   425  		// Handle exact integer value.
   426  		switch t.str[0] {
   427  		case 'u':
   428  			return t.num
   429  		case 'i':
   430  			if int64(t.num) < minUint64 {
   431  				return minUint64
   432  			}
   433  			return uint64(int64(t.num))
   434  		}
   435  	}
   436  
   437  	// Handle JSON number that is a floating-point value.
   438  	if t.Kind() == '0' {
   439  		switch fv := t.Float(); {
   440  		case fv >= maxUint64:
   441  			return maxUint64
   442  		case fv <= minUint64:
   443  			return minUint64
   444  		default:
   445  			return uint64(fv) // truncation toward zero
   446  		}
   447  	}
   448  
   449  	panic("invalid JSON token kind: " + t.Kind().String())
   450  }
   451  
   452  // Kind returns the token kind.
   453  func (t Token) Kind() Kind {
   454  	switch {
   455  	case t.raw != nil:
   456  		raw := t.raw
   457  		if uint64(raw.previousOffsetStart()) != t.num {
   458  			panic(invalidTokenPanic)
   459  		}
   460  		return Kind(t.raw.buf[raw.prevStart]).normalize()
   461  	case t.num != 0:
   462  		return '0'
   463  	case len(t.str) != 0:
   464  		return '"'
   465  	default:
   466  		return invalidKind
   467  	}
   468  }
   469  
   470  // Kind represents each possible JSON token kind with a single byte,
   471  // which is conveniently the first byte of that kind's grammar
   472  // with the restriction that numbers always be represented with '0':
   473  //
   474  //   - 'n': null
   475  //   - 'f': false
   476  //   - 't': true
   477  //   - '"': string
   478  //   - '0': number
   479  //   - '{': object start
   480  //   - '}': object end
   481  //   - '[': array start
   482  //   - ']': array end
   483  //
   484  // An invalid kind is usually represented using 0,
   485  // but may be non-zero due to invalid JSON data.
   486  type Kind byte
   487  
   488  const invalidKind Kind = 0
   489  
   490  // String prints the kind in a humanly readable fashion.
   491  func (k Kind) String() string {
   492  	switch k {
   493  	case 'n':
   494  		return "null"
   495  	case 'f':
   496  		return "false"
   497  	case 't':
   498  		return "true"
   499  	case '"':
   500  		return "string"
   501  	case '0':
   502  		return "number"
   503  	case '{':
   504  		return "{"
   505  	case '}':
   506  		return "}"
   507  	case '[':
   508  		return "["
   509  	case ']':
   510  		return "]"
   511  	default:
   512  		return "<invalid json.Kind: " + quoteRune([]byte{byte(k)}) + ">"
   513  	}
   514  }
   515  
   516  // normalize coalesces all possible starting characters of a number as just '0'.
   517  func (k Kind) normalize() Kind {
   518  	if k == '-' || ('0' <= k && k <= '9') {
   519  		return '0'
   520  	}
   521  	return k
   522  }
   523
View as plain text