token.go

Documentation: cuelang.org/go/cue/token

     1  // Copyright 2018 The CUE Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  // Package token defines constants representing the lexical tokens of the Go
    16  // programming language and basic operations on tokens (printing, predicates).
    17  package token // import "cuelang.org/go/cue/token"
    18  
    19  import "strconv"
    20  
    21  // Token is the set of lexical tokens of the CUE configuration language.
    22  type Token int
    23  
    24  // The list of tokens.
    25  const (
    26  	// Special tokens
    27  	ILLEGAL Token = iota
    28  	EOF
    29  	COMMENT
    30  	ATTRIBUTE // @foo(bar,baz=4)
    31  
    32  	literalBeg
    33  	// Identifiers and basic type literals
    34  	// (these tokens stand for classes of literals)
    35  	IDENT // main, _tmp
    36  	INT   // 12_345Mi, 0700, 0xdeadbeef, 1.2M
    37  	FLOAT // 123.45,
    38  	// DURATION // 3m4s TODO
    39  	STRING        // "abc"
    40  	INTERPOLATION // a part of a template string, e.g. `"age: \(`
    41  	BOTTOM        // _|_
    42  
    43  	literalEnd
    44  
    45  	operatorBeg
    46  	// Operators and delimiters
    47  	ADD // +
    48  	SUB // -
    49  	MUL // *
    50  	POW // ^
    51  	QUO // /
    52  
    53  	IQUO // quo
    54  	IREM // rem
    55  	IDIV // div
    56  	IMOD // mod
    57  
    58  	AND // &
    59  	OR  // |
    60  
    61  	LAND // &&
    62  	LOR  // ||
    63  
    64  	BIND  // =
    65  	EQL   // ==
    66  	LSS   // <
    67  	GTR   // >
    68  	NOT   // !
    69  	ARROW // <-
    70  
    71  	NEQ // !=
    72  	LEQ // <=
    73  	GEQ // >=
    74  
    75  	MAT  // =~
    76  	NMAT // !~
    77  
    78  	LPAREN   // (
    79  	LBRACK   // [
    80  	LBRACE   // {
    81  	COMMA    // ,
    82  	PERIOD   // .
    83  	ELLIPSIS // ...
    84  
    85  	RPAREN    // )
    86  	RBRACK    // ]
    87  	RBRACE    // }
    88  	SEMICOLON // ;
    89  	COLON     // :
    90  	OPTION    // ?
    91  	operatorEnd
    92  
    93  	keywordBeg
    94  
    95  	IF
    96  	FOR
    97  	IN
    98  	LET
    99  	FUNC // experimental
   100  
   101  	TRUE
   102  	FALSE
   103  	NULL
   104  
   105  	keywordEnd
   106  )
   107  
   108  var tokens = [...]string{
   109  	ILLEGAL: "ILLEGAL",
   110  
   111  	EOF:     "EOF",
   112  	COMMENT: "COMMENT",
   113  
   114  	IDENT:         "IDENT",
   115  	INT:           "INT",
   116  	FLOAT:         "FLOAT",
   117  	STRING:        "STRING",
   118  	INTERPOLATION: "INTERPOLATION",
   119  	ATTRIBUTE:     "ATTRIBUTE",
   120  
   121  	ADD: "+",
   122  	SUB: "-",
   123  	MUL: "*",
   124  	POW: "^",
   125  	QUO: "/",
   126  
   127  	IQUO: "quo",
   128  	IREM: "rem",
   129  	IDIV: "div",
   130  	IMOD: "mod",
   131  
   132  	AND: "&",
   133  	OR:  "|",
   134  
   135  	LAND: "&&",
   136  	LOR:  "||",
   137  
   138  	BIND:  "=",
   139  	EQL:   "==",
   140  	LSS:   "<",
   141  	GTR:   ">",
   142  	NOT:   "!",
   143  	ARROW: "<-",
   144  
   145  	NEQ: "!=",
   146  	LEQ: "<=",
   147  	GEQ: ">=",
   148  
   149  	MAT:  "=~",
   150  	NMAT: "!~",
   151  
   152  	LPAREN:   "(",
   153  	LBRACK:   "[",
   154  	LBRACE:   "{",
   155  	COMMA:    ",",
   156  	PERIOD:   ".",
   157  	ELLIPSIS: "...",
   158  
   159  	RPAREN:    ")",
   160  	RBRACK:    "]",
   161  	RBRACE:    "}",
   162  	SEMICOLON: ";",
   163  	COLON:     ":",
   164  	OPTION:    "?",
   165  
   166  	BOTTOM: "_|_",
   167  
   168  	FALSE: "false",
   169  	TRUE:  "true",
   170  	NULL:  "null",
   171  
   172  	FOR:  "for",
   173  	IF:   "if",
   174  	IN:   "in",
   175  	LET:  "let",
   176  	FUNC: "func",
   177  }
   178  
   179  // String returns the string corresponding to the token tok.
   180  // For operators, delimiters, and keywords the string is the actual
   181  // token character sequence (e.g., for the token ADD, the string is
   182  // "+"). For all other tokens the string corresponds to the token
   183  // constant name (e.g. for the token IDENT, the string is "IDENT").
   184  func (tok Token) String() string {
   185  	s := ""
   186  	if 0 <= tok && tok < Token(len(tokens)) {
   187  		s = tokens[tok]
   188  	}
   189  	if s == "" {
   190  		s = "token(" + strconv.Itoa(int(tok)) + ")"
   191  	}
   192  	return s
   193  }
   194  
   195  // A set of constants for precedence-based expression parsing.
   196  // Non-operators have lowest precedence, followed by operators
   197  // starting with precedence 1 up to unary operators. The highest
   198  // precedence serves as "catch-all" precedence for selector,
   199  // indexing, and other operator and delimiter tokens.
   200  const (
   201  	LowestPrec  = lowestPrec
   202  	UnaryPrec   = unaryPrec
   203  	HighestPrec = highestPrec
   204  )
   205  
   206  const (
   207  	lowestPrec  = 0 // non-operators
   208  	unaryPrec   = 8
   209  	highestPrec = 9
   210  )
   211  
   212  // Precedence returns the operator precedence of the binary
   213  // operator op. If op is not a binary operator, the result
   214  // is LowestPrecedence.
   215  func (tok Token) Precedence() int {
   216  	switch tok {
   217  	case OR:
   218  		return 1
   219  	case AND:
   220  		return 2
   221  	case LOR:
   222  		return 3
   223  	case LAND:
   224  		return 4
   225  	case EQL, NEQ, LSS, LEQ, GTR, GEQ, MAT, NMAT:
   226  		return 5
   227  	case ADD, SUB:
   228  		return 6
   229  	case MUL, QUO, IDIV, IMOD, IQUO, IREM:
   230  		return 7
   231  	}
   232  	return lowestPrec
   233  }
   234  
   235  var keywords map[string]Token
   236  
   237  func init() {
   238  	keywords = make(map[string]Token)
   239  	for i := keywordBeg + 1; i < keywordEnd; i++ {
   240  		keywords[tokens[i]] = i
   241  	}
   242  }
   243  
   244  // Lookup maps an identifier to its keyword token or IDENT (if not a keyword).
   245  func Lookup(ident string) Token {
   246  	if tok, isKeyword := keywords[ident]; isKeyword {
   247  		return tok
   248  	}
   249  	return IDENT
   250  }
   251  
   252  // Predicates
   253  
   254  // IsLiteral returns true for tokens corresponding to identifiers
   255  // and basic type literals; it returns false otherwise.
   256  func (tok Token) IsLiteral() bool { return literalBeg < tok && tok < literalEnd }
   257  
   258  // IsOperator returns true for tokens corresponding to operators and
   259  // delimiters; it returns false otherwise.
   260  func (tok Token) IsOperator() bool { return operatorBeg < tok && tok < operatorEnd }
   261  
   262  // IsKeyword returns true for tokens corresponding to keywords;
   263  // it returns false otherwise.
   264  func (tok Token) IsKeyword() bool { return keywordBeg < tok && tok < keywordEnd }
   265
View as plain text