shlex.go

Documentation: github.com/google/shlex

     1  /*
     2  Copyright 2012 Google Inc. All Rights Reserved.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  /*
    18  Package shlex implements a simple lexer which splits input in to tokens using
    19  shell-style rules for quoting and commenting.
    20  
    21  The basic use case uses the default ASCII lexer to split a string into sub-strings:
    22  
    23    shlex.Split("one \"two three\" four") -> []string{"one", "two three", "four"}
    24  
    25  To process a stream of strings:
    26  
    27    l := NewLexer(os.Stdin)
    28    for ; token, err := l.Next(); err != nil {
    29    	// process token
    30    }
    31  
    32  To access the raw token stream (which includes tokens for comments):
    33  
    34    t := NewTokenizer(os.Stdin)
    35    for ; token, err := t.Next(); err != nil {
    36  	// process token
    37    }
    38  
    39  */
    40  package shlex
    41  
    42  import (
    43  	"bufio"
    44  	"fmt"
    45  	"io"
    46  	"strings"
    47  )
    48  
    49  // TokenType is a top-level token classification: A word, space, comment, unknown.
    50  type TokenType int
    51  
    52  // runeTokenClass is the type of a UTF-8 character classification: A quote, space, escape.
    53  type runeTokenClass int
    54  
    55  // the internal state used by the lexer state machine
    56  type lexerState int
    57  
    58  // Token is a (type, value) pair representing a lexographical token.
    59  type Token struct {
    60  	tokenType TokenType
    61  	value     string
    62  }
    63  
    64  // Equal reports whether tokens a, and b, are equal.
    65  // Two tokens are equal if both their types and values are equal. A nil token can
    66  // never be equal to another token.
    67  func (a *Token) Equal(b *Token) bool {
    68  	if a == nil || b == nil {
    69  		return false
    70  	}
    71  	if a.tokenType != b.tokenType {
    72  		return false
    73  	}
    74  	return a.value == b.value
    75  }
    76  
    77  // Named classes of UTF-8 runes
    78  const (
    79  	spaceRunes            = " \t\r\n"
    80  	escapingQuoteRunes    = `"`
    81  	nonEscapingQuoteRunes = "'"
    82  	escapeRunes           = `\`
    83  	commentRunes          = "#"
    84  )
    85  
    86  // Classes of rune token
    87  const (
    88  	unknownRuneClass runeTokenClass = iota
    89  	spaceRuneClass
    90  	escapingQuoteRuneClass
    91  	nonEscapingQuoteRuneClass
    92  	escapeRuneClass
    93  	commentRuneClass
    94  	eofRuneClass
    95  )
    96  
    97  // Classes of lexographic token
    98  const (
    99  	UnknownToken TokenType = iota
   100  	WordToken
   101  	SpaceToken
   102  	CommentToken
   103  )
   104  
   105  // Lexer state machine states
   106  const (
   107  	startState           lexerState = iota // no runes have been seen
   108  	inWordState                            // processing regular runes in a word
   109  	escapingState                          // we have just consumed an escape rune; the next rune is literal
   110  	escapingQuotedState                    // we have just consumed an escape rune within a quoted string
   111  	quotingEscapingState                   // we are within a quoted string that supports escaping ("...")
   112  	quotingState                           // we are within a string that does not support escaping ('...')
   113  	commentState                           // we are within a comment (everything following an unquoted or unescaped #
   114  )
   115  
   116  // tokenClassifier is used for classifying rune characters.
   117  type tokenClassifier map[rune]runeTokenClass
   118  
   119  func (typeMap tokenClassifier) addRuneClass(runes string, tokenType runeTokenClass) {
   120  	for _, runeChar := range runes {
   121  		typeMap[runeChar] = tokenType
   122  	}
   123  }
   124  
   125  // newDefaultClassifier creates a new classifier for ASCII characters.
   126  func newDefaultClassifier() tokenClassifier {
   127  	t := tokenClassifier{}
   128  	t.addRuneClass(spaceRunes, spaceRuneClass)
   129  	t.addRuneClass(escapingQuoteRunes, escapingQuoteRuneClass)
   130  	t.addRuneClass(nonEscapingQuoteRunes, nonEscapingQuoteRuneClass)
   131  	t.addRuneClass(escapeRunes, escapeRuneClass)
   132  	t.addRuneClass(commentRunes, commentRuneClass)
   133  	return t
   134  }
   135  
   136  // ClassifyRune classifiees a rune
   137  func (t tokenClassifier) ClassifyRune(runeVal rune) runeTokenClass {
   138  	return t[runeVal]
   139  }
   140  
   141  // Lexer turns an input stream into a sequence of tokens. Whitespace and comments are skipped.
   142  type Lexer Tokenizer
   143  
   144  // NewLexer creates a new lexer from an input stream.
   145  func NewLexer(r io.Reader) *Lexer {
   146  
   147  	return (*Lexer)(NewTokenizer(r))
   148  }
   149  
   150  // Next returns the next word, or an error. If there are no more words,
   151  // the error will be io.EOF.
   152  func (l *Lexer) Next() (string, error) {
   153  	for {
   154  		token, err := (*Tokenizer)(l).Next()
   155  		if err != nil {
   156  			return "", err
   157  		}
   158  		switch token.tokenType {
   159  		case WordToken:
   160  			return token.value, nil
   161  		case CommentToken:
   162  			// skip comments
   163  		default:
   164  			return "", fmt.Errorf("Unknown token type: %v", token.tokenType)
   165  		}
   166  	}
   167  }
   168  
   169  // Tokenizer turns an input stream into a sequence of typed tokens
   170  type Tokenizer struct {
   171  	input      bufio.Reader
   172  	classifier tokenClassifier
   173  }
   174  
   175  // NewTokenizer creates a new tokenizer from an input stream.
   176  func NewTokenizer(r io.Reader) *Tokenizer {
   177  	input := bufio.NewReader(r)
   178  	classifier := newDefaultClassifier()
   179  	return &Tokenizer{
   180  		input:      *input,
   181  		classifier: classifier}
   182  }
   183  
   184  // scanStream scans the stream for the next token using the internal state machine.
   185  // It will panic if it encounters a rune which it does not know how to handle.
   186  func (t *Tokenizer) scanStream() (*Token, error) {
   187  	state := startState
   188  	var tokenType TokenType
   189  	var value []rune
   190  	var nextRune rune
   191  	var nextRuneType runeTokenClass
   192  	var err error
   193  
   194  	for {
   195  		nextRune, _, err = t.input.ReadRune()
   196  		nextRuneType = t.classifier.ClassifyRune(nextRune)
   197  
   198  		if err == io.EOF {
   199  			nextRuneType = eofRuneClass
   200  			err = nil
   201  		} else if err != nil {
   202  			return nil, err
   203  		}
   204  
   205  		switch state {
   206  		case startState: // no runes read yet
   207  			{
   208  				switch nextRuneType {
   209  				case eofRuneClass:
   210  					{
   211  						return nil, io.EOF
   212  					}
   213  				case spaceRuneClass:
   214  					{
   215  					}
   216  				case escapingQuoteRuneClass:
   217  					{
   218  						tokenType = WordToken
   219  						state = quotingEscapingState
   220  					}
   221  				case nonEscapingQuoteRuneClass:
   222  					{
   223  						tokenType = WordToken
   224  						state = quotingState
   225  					}
   226  				case escapeRuneClass:
   227  					{
   228  						tokenType = WordToken
   229  						state = escapingState
   230  					}
   231  				case commentRuneClass:
   232  					{
   233  						tokenType = CommentToken
   234  						state = commentState
   235  					}
   236  				default:
   237  					{
   238  						tokenType = WordToken
   239  						value = append(value, nextRune)
   240  						state = inWordState
   241  					}
   242  				}
   243  			}
   244  		case inWordState: // in a regular word
   245  			{
   246  				switch nextRuneType {
   247  				case eofRuneClass:
   248  					{
   249  						token := &Token{
   250  							tokenType: tokenType,
   251  							value:     string(value)}
   252  						return token, err
   253  					}
   254  				case spaceRuneClass:
   255  					{
   256  						token := &Token{
   257  							tokenType: tokenType,
   258  							value:     string(value)}
   259  						return token, err
   260  					}
   261  				case escapingQuoteRuneClass:
   262  					{
   263  						state = quotingEscapingState
   264  					}
   265  				case nonEscapingQuoteRuneClass:
   266  					{
   267  						state = quotingState
   268  					}
   269  				case escapeRuneClass:
   270  					{
   271  						state = escapingState
   272  					}
   273  				default:
   274  					{
   275  						value = append(value, nextRune)
   276  					}
   277  				}
   278  			}
   279  		case escapingState: // the rune after an escape character
   280  			{
   281  				switch nextRuneType {
   282  				case eofRuneClass:
   283  					{
   284  						err = fmt.Errorf("EOF found after escape character")
   285  						token := &Token{
   286  							tokenType: tokenType,
   287  							value:     string(value)}
   288  						return token, err
   289  					}
   290  				default:
   291  					{
   292  						state = inWordState
   293  						value = append(value, nextRune)
   294  					}
   295  				}
   296  			}
   297  		case escapingQuotedState: // the next rune after an escape character, in double quotes
   298  			{
   299  				switch nextRuneType {
   300  				case eofRuneClass:
   301  					{
   302  						err = fmt.Errorf("EOF found after escape character")
   303  						token := &Token{
   304  							tokenType: tokenType,
   305  							value:     string(value)}
   306  						return token, err
   307  					}
   308  				default:
   309  					{
   310  						state = quotingEscapingState
   311  						value = append(value, nextRune)
   312  					}
   313  				}
   314  			}
   315  		case quotingEscapingState: // in escaping double quotes
   316  			{
   317  				switch nextRuneType {
   318  				case eofRuneClass:
   319  					{
   320  						err = fmt.Errorf("EOF found when expecting closing quote")
   321  						token := &Token{
   322  							tokenType: tokenType,
   323  							value:     string(value)}
   324  						return token, err
   325  					}
   326  				case escapingQuoteRuneClass:
   327  					{
   328  						state = inWordState
   329  					}
   330  				case escapeRuneClass:
   331  					{
   332  						state = escapingQuotedState
   333  					}
   334  				default:
   335  					{
   336  						value = append(value, nextRune)
   337  					}
   338  				}
   339  			}
   340  		case quotingState: // in non-escaping single quotes
   341  			{
   342  				switch nextRuneType {
   343  				case eofRuneClass:
   344  					{
   345  						err = fmt.Errorf("EOF found when expecting closing quote")
   346  						token := &Token{
   347  							tokenType: tokenType,
   348  							value:     string(value)}
   349  						return token, err
   350  					}
   351  				case nonEscapingQuoteRuneClass:
   352  					{
   353  						state = inWordState
   354  					}
   355  				default:
   356  					{
   357  						value = append(value, nextRune)
   358  					}
   359  				}
   360  			}
   361  		case commentState: // in a comment
   362  			{
   363  				switch nextRuneType {
   364  				case eofRuneClass:
   365  					{
   366  						token := &Token{
   367  							tokenType: tokenType,
   368  							value:     string(value)}
   369  						return token, err
   370  					}
   371  				case spaceRuneClass:
   372  					{
   373  						if nextRune == '\n' {
   374  							state = startState
   375  							token := &Token{
   376  								tokenType: tokenType,
   377  								value:     string(value)}
   378  							return token, err
   379  						} else {
   380  							value = append(value, nextRune)
   381  						}
   382  					}
   383  				default:
   384  					{
   385  						value = append(value, nextRune)
   386  					}
   387  				}
   388  			}
   389  		default:
   390  			{
   391  				return nil, fmt.Errorf("Unexpected state: %v", state)
   392  			}
   393  		}
   394  	}
   395  }
   396  
   397  // Next returns the next token in the stream.
   398  func (t *Tokenizer) Next() (*Token, error) {
   399  	return t.scanStream()
   400  }
   401  
   402  // Split partitions a string into a slice of strings.
   403  func Split(s string) ([]string, error) {
   404  	l := NewLexer(strings.NewReader(s))
   405  	subStrings := make([]string, 0)
   406  	for {
   407  		word, err := l.Next()
   408  		if err != nil {
   409  			if err == io.EOF {
   410  				return subStrings, nil
   411  			}
   412  			return subStrings, err
   413  		}
   414  		subStrings = append(subStrings, word)
   415  	}
   416  }
   417
View as plain text