token.go

Documentation: github.com/pelletier/go-toml

     1  package toml
     2  
     3  import "fmt"
     4  
     5  // Define tokens
     6  type tokenType int
     7  
     8  const (
     9  	eof = -(iota + 1)
    10  )
    11  
    12  const (
    13  	tokenError tokenType = iota
    14  	tokenEOF
    15  	tokenComment
    16  	tokenKey
    17  	tokenString
    18  	tokenInteger
    19  	tokenTrue
    20  	tokenFalse
    21  	tokenFloat
    22  	tokenInf
    23  	tokenNan
    24  	tokenEqual
    25  	tokenLeftBracket
    26  	tokenRightBracket
    27  	tokenLeftCurlyBrace
    28  	tokenRightCurlyBrace
    29  	tokenLeftParen
    30  	tokenRightParen
    31  	tokenDoubleLeftBracket
    32  	tokenDoubleRightBracket
    33  	tokenLocalDate
    34  	tokenLocalTime
    35  	tokenTimeOffset
    36  	tokenKeyGroup
    37  	tokenKeyGroupArray
    38  	tokenComma
    39  	tokenColon
    40  	tokenDollar
    41  	tokenStar
    42  	tokenQuestion
    43  	tokenDot
    44  	tokenDotDot
    45  	tokenEOL
    46  )
    47  
    48  var tokenTypeNames = []string{
    49  	"Error",
    50  	"EOF",
    51  	"Comment",
    52  	"Key",
    53  	"String",
    54  	"Integer",
    55  	"True",
    56  	"False",
    57  	"Float",
    58  	"Inf",
    59  	"NaN",
    60  	"=",
    61  	"[",
    62  	"]",
    63  	"{",
    64  	"}",
    65  	"(",
    66  	")",
    67  	"]]",
    68  	"[[",
    69  	"LocalDate",
    70  	"LocalTime",
    71  	"TimeOffset",
    72  	"KeyGroup",
    73  	"KeyGroupArray",
    74  	",",
    75  	":",
    76  	"$",
    77  	"*",
    78  	"?",
    79  	".",
    80  	"..",
    81  	"EOL",
    82  }
    83  
    84  type token struct {
    85  	Position
    86  	typ tokenType
    87  	val string
    88  }
    89  
    90  func (tt tokenType) String() string {
    91  	idx := int(tt)
    92  	if idx < len(tokenTypeNames) {
    93  		return tokenTypeNames[idx]
    94  	}
    95  	return "Unknown"
    96  }
    97  
    98  func (t token) String() string {
    99  	switch t.typ {
   100  	case tokenEOF:
   101  		return "EOF"
   102  	case tokenError:
   103  		return t.val
   104  	}
   105  
   106  	return fmt.Sprintf("%q", t.val)
   107  }
   108  
   109  func isSpace(r rune) bool {
   110  	return r == ' ' || r == '\t'
   111  }
   112  
   113  func isAlphanumeric(r rune) bool {
   114  	return 'a' <= r && r <= 'z' || 'A' <= r && r <= 'Z' || r == '_'
   115  }
   116  
   117  func isKeyChar(r rune) bool {
   118  	// Keys start with the first character that isn't whitespace or [ and end
   119  	// with the last non-whitespace character before the equals sign. Keys
   120  	// cannot contain a # character."
   121  	return !(r == '\r' || r == '\n' || r == eof || r == '=')
   122  }
   123  
   124  func isKeyStartChar(r rune) bool {
   125  	return !(isSpace(r) || r == '\r' || r == '\n' || r == eof || r == '[')
   126  }
   127  
   128  func isDigit(r rune) bool {
   129  	return '0' <= r && r <= '9'
   130  }
   131  
   132  func isHexDigit(r rune) bool {
   133  	return isDigit(r) ||
   134  		(r >= 'a' && r <= 'f') ||
   135  		(r >= 'A' && r <= 'F')
   136  }
   137
View as plain text