iterator.go

Documentation: github.com/alecthomas/chroma

     1  package chroma
     2  
     3  import "strings"
     4  
     5  // An Iterator across tokens.
     6  //
     7  // EOF will be returned at the end of the Token stream.
     8  //
     9  // If an error occurs within an Iterator, it may propagate this in a panic. Formatters should recover.
    10  type Iterator func() Token
    11  
    12  // Tokens consumes all tokens from the iterator and returns them as a slice.
    13  func (i Iterator) Tokens() []Token {
    14  	var out []Token
    15  	for t := i(); t != EOF; t = i() {
    16  		out = append(out, t)
    17  	}
    18  	return out
    19  }
    20  
    21  // Concaterator concatenates tokens from a series of iterators.
    22  func Concaterator(iterators ...Iterator) Iterator {
    23  	return func() Token {
    24  		for len(iterators) > 0 {
    25  			t := iterators[0]()
    26  			if t != EOF {
    27  				return t
    28  			}
    29  			iterators = iterators[1:]
    30  		}
    31  		return EOF
    32  	}
    33  }
    34  
    35  // Literator converts a sequence of literal Tokens into an Iterator.
    36  func Literator(tokens ...Token) Iterator {
    37  	return func() Token {
    38  		if len(tokens) == 0 {
    39  			return EOF
    40  		}
    41  		token := tokens[0]
    42  		tokens = tokens[1:]
    43  		return token
    44  	}
    45  }
    46  
    47  // SplitTokensIntoLines splits tokens containing newlines in two.
    48  func SplitTokensIntoLines(tokens []Token) (out [][]Token) {
    49  	var line []Token // nolint: prealloc
    50  	for _, token := range tokens {
    51  		for strings.Contains(token.Value, "\n") {
    52  			parts := strings.SplitAfterN(token.Value, "\n", 2)
    53  			// Token becomes the tail.
    54  			token.Value = parts[1]
    55  
    56  			// Append the head to the line and flush the line.
    57  			clone := token.Clone()
    58  			clone.Value = parts[0]
    59  			line = append(line, clone)
    60  			out = append(out, line)
    61  			line = nil
    62  		}
    63  		line = append(line, token)
    64  	}
    65  	if len(line) > 0 {
    66  		out = append(out, line)
    67  	}
    68  	// Strip empty trailing token line.
    69  	if len(out) > 0 {
    70  		last := out[len(out)-1]
    71  		if len(last) == 1 && last[0].Value == "" {
    72  			out = out[:len(out)-1]
    73  		}
    74  	}
    75  	return
    76  }
    77
View as plain text