parser.go

Documentation: gopkg.in/ini.v1

     1  // Copyright 2015 Unknwon
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License"): you may
     4  // not use this file except in compliance with the License. You may obtain
     5  // a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
    11  // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
    12  // License for the specific language governing permissions and limitations
    13  // under the License.
    14  
    15  package ini
    16  
    17  import (
    18  	"bufio"
    19  	"bytes"
    20  	"fmt"
    21  	"io"
    22  	"regexp"
    23  	"strconv"
    24  	"strings"
    25  	"unicode"
    26  )
    27  
    28  const minReaderBufferSize = 4096
    29  
    30  var pythonMultiline = regexp.MustCompile(`^([\t\f ]+)(.*)`)
    31  
    32  type parserOptions struct {
    33  	IgnoreContinuation          bool
    34  	IgnoreInlineComment         bool
    35  	AllowPythonMultilineValues  bool
    36  	SpaceBeforeInlineComment    bool
    37  	UnescapeValueDoubleQuotes   bool
    38  	UnescapeValueCommentSymbols bool
    39  	PreserveSurroundedQuote     bool
    40  	DebugFunc                   DebugFunc
    41  	ReaderBufferSize            int
    42  }
    43  
    44  type parser struct {
    45  	buf     *bufio.Reader
    46  	options parserOptions
    47  
    48  	isEOF   bool
    49  	count   int
    50  	comment *bytes.Buffer
    51  }
    52  
    53  func (p *parser) debug(format string, args ...interface{}) {
    54  	if p.options.DebugFunc != nil {
    55  		p.options.DebugFunc(fmt.Sprintf(format, args...))
    56  	}
    57  }
    58  
    59  func newParser(r io.Reader, opts parserOptions) *parser {
    60  	size := opts.ReaderBufferSize
    61  	if size < minReaderBufferSize {
    62  		size = minReaderBufferSize
    63  	}
    64  
    65  	return &parser{
    66  		buf:     bufio.NewReaderSize(r, size),
    67  		options: opts,
    68  		count:   1,
    69  		comment: &bytes.Buffer{},
    70  	}
    71  }
    72  
    73  // BOM handles header of UTF-8, UTF-16 LE and UTF-16 BE's BOM format.
    74  // http://en.wikipedia.org/wiki/Byte_order_mark#Representations_of_byte_order_marks_by_encoding
    75  func (p *parser) BOM() error {
    76  	mask, err := p.buf.Peek(2)
    77  	if err != nil && err != io.EOF {
    78  		return err
    79  	} else if len(mask) < 2 {
    80  		return nil
    81  	}
    82  
    83  	switch {
    84  	case mask[0] == 254 && mask[1] == 255:
    85  		fallthrough
    86  	case mask[0] == 255 && mask[1] == 254:
    87  		_, err = p.buf.Read(mask)
    88  		if err != nil {
    89  			return err
    90  		}
    91  	case mask[0] == 239 && mask[1] == 187:
    92  		mask, err := p.buf.Peek(3)
    93  		if err != nil && err != io.EOF {
    94  			return err
    95  		} else if len(mask) < 3 {
    96  			return nil
    97  		}
    98  		if mask[2] == 191 {
    99  			_, err = p.buf.Read(mask)
   100  			if err != nil {
   101  				return err
   102  			}
   103  		}
   104  	}
   105  	return nil
   106  }
   107  
   108  func (p *parser) readUntil(delim byte) ([]byte, error) {
   109  	data, err := p.buf.ReadBytes(delim)
   110  	if err != nil {
   111  		if err == io.EOF {
   112  			p.isEOF = true
   113  		} else {
   114  			return nil, err
   115  		}
   116  	}
   117  	return data, nil
   118  }
   119  
   120  func cleanComment(in []byte) ([]byte, bool) {
   121  	i := bytes.IndexAny(in, "#;")
   122  	if i == -1 {
   123  		return nil, false
   124  	}
   125  	return in[i:], true
   126  }
   127  
   128  func readKeyName(delimiters string, in []byte) (string, int, error) {
   129  	line := string(in)
   130  
   131  	// Check if key name surrounded by quotes.
   132  	var keyQuote string
   133  	if line[0] == '"' {
   134  		if len(line) > 6 && line[0:3] == `"""` {
   135  			keyQuote = `"""`
   136  		} else {
   137  			keyQuote = `"`
   138  		}
   139  	} else if line[0] == '`' {
   140  		keyQuote = "`"
   141  	}
   142  
   143  	// Get out key name
   144  	var endIdx int
   145  	if len(keyQuote) > 0 {
   146  		startIdx := len(keyQuote)
   147  		// FIXME: fail case -> """"""name"""=value
   148  		pos := strings.Index(line[startIdx:], keyQuote)
   149  		if pos == -1 {
   150  			return "", -1, fmt.Errorf("missing closing key quote: %s", line)
   151  		}
   152  		pos += startIdx
   153  
   154  		// Find key-value delimiter
   155  		i := strings.IndexAny(line[pos+startIdx:], delimiters)
   156  		if i < 0 {
   157  			return "", -1, ErrDelimiterNotFound{line}
   158  		}
   159  		endIdx = pos + i
   160  		return strings.TrimSpace(line[startIdx:pos]), endIdx + startIdx + 1, nil
   161  	}
   162  
   163  	endIdx = strings.IndexAny(line, delimiters)
   164  	if endIdx < 0 {
   165  		return "", -1, ErrDelimiterNotFound{line}
   166  	}
   167  	if endIdx == 0 {
   168  		return "", -1, ErrEmptyKeyName{line}
   169  	}
   170  
   171  	return strings.TrimSpace(line[0:endIdx]), endIdx + 1, nil
   172  }
   173  
   174  func (p *parser) readMultilines(line, val, valQuote string) (string, error) {
   175  	for {
   176  		data, err := p.readUntil('\n')
   177  		if err != nil {
   178  			return "", err
   179  		}
   180  		next := string(data)
   181  
   182  		pos := strings.LastIndex(next, valQuote)
   183  		if pos > -1 {
   184  			val += next[:pos]
   185  
   186  			comment, has := cleanComment([]byte(next[pos:]))
   187  			if has {
   188  				p.comment.Write(bytes.TrimSpace(comment))
   189  			}
   190  			break
   191  		}
   192  		val += next
   193  		if p.isEOF {
   194  			return "", fmt.Errorf("missing closing key quote from %q to %q", line, next)
   195  		}
   196  	}
   197  	return val, nil
   198  }
   199  
   200  func (p *parser) readContinuationLines(val string) (string, error) {
   201  	for {
   202  		data, err := p.readUntil('\n')
   203  		if err != nil {
   204  			return "", err
   205  		}
   206  		next := strings.TrimSpace(string(data))
   207  
   208  		if len(next) == 0 {
   209  			break
   210  		}
   211  		val += next
   212  		if val[len(val)-1] != '\\' {
   213  			break
   214  		}
   215  		val = val[:len(val)-1]
   216  	}
   217  	return val, nil
   218  }
   219  
   220  // hasSurroundedQuote check if and only if the first and last characters
   221  // are quotes \" or \'.
   222  // It returns false if any other parts also contain same kind of quotes.
   223  func hasSurroundedQuote(in string, quote byte) bool {
   224  	return len(in) >= 2 && in[0] == quote && in[len(in)-1] == quote &&
   225  		strings.IndexByte(in[1:], quote) == len(in)-2
   226  }
   227  
   228  func (p *parser) readValue(in []byte, bufferSize int) (string, error) {
   229  
   230  	line := strings.TrimLeftFunc(string(in), unicode.IsSpace)
   231  	if len(line) == 0 {
   232  		if p.options.AllowPythonMultilineValues && len(in) > 0 && in[len(in)-1] == '\n' {
   233  			return p.readPythonMultilines(line, bufferSize)
   234  		}
   235  		return "", nil
   236  	}
   237  
   238  	var valQuote string
   239  	if len(line) > 3 && line[0:3] == `"""` {
   240  		valQuote = `"""`
   241  	} else if line[0] == '`' {
   242  		valQuote = "`"
   243  	} else if p.options.UnescapeValueDoubleQuotes && line[0] == '"' {
   244  		valQuote = `"`
   245  	}
   246  
   247  	if len(valQuote) > 0 {
   248  		startIdx := len(valQuote)
   249  		pos := strings.LastIndex(line[startIdx:], valQuote)
   250  		// Check for multi-line value
   251  		if pos == -1 {
   252  			return p.readMultilines(line, line[startIdx:], valQuote)
   253  		}
   254  
   255  		if p.options.UnescapeValueDoubleQuotes && valQuote == `"` {
   256  			return strings.Replace(line[startIdx:pos+startIdx], `\"`, `"`, -1), nil
   257  		}
   258  		return line[startIdx : pos+startIdx], nil
   259  	}
   260  
   261  	lastChar := line[len(line)-1]
   262  	// Won't be able to reach here if value only contains whitespace
   263  	line = strings.TrimSpace(line)
   264  	trimmedLastChar := line[len(line)-1]
   265  
   266  	// Check continuation lines when desired
   267  	if !p.options.IgnoreContinuation && trimmedLastChar == '\\' {
   268  		return p.readContinuationLines(line[:len(line)-1])
   269  	}
   270  
   271  	// Check if ignore inline comment
   272  	if !p.options.IgnoreInlineComment {
   273  		var i int
   274  		if p.options.SpaceBeforeInlineComment {
   275  			i = strings.Index(line, " #")
   276  			if i == -1 {
   277  				i = strings.Index(line, " ;")
   278  			}
   279  
   280  		} else {
   281  			i = strings.IndexAny(line, "#;")
   282  		}
   283  
   284  		if i > -1 {
   285  			p.comment.WriteString(line[i:])
   286  			line = strings.TrimSpace(line[:i])
   287  		}
   288  
   289  	}
   290  
   291  	// Trim single and double quotes
   292  	if (hasSurroundedQuote(line, '\'') ||
   293  		hasSurroundedQuote(line, '"')) && !p.options.PreserveSurroundedQuote {
   294  		line = line[1 : len(line)-1]
   295  	} else if len(valQuote) == 0 && p.options.UnescapeValueCommentSymbols {
   296  		line = strings.ReplaceAll(line, `\;`, ";")
   297  		line = strings.ReplaceAll(line, `\#`, "#")
   298  	} else if p.options.AllowPythonMultilineValues && lastChar == '\n' {
   299  		return p.readPythonMultilines(line, bufferSize)
   300  	}
   301  
   302  	return line, nil
   303  }
   304  
   305  func (p *parser) readPythonMultilines(line string, bufferSize int) (string, error) {
   306  	parserBufferPeekResult, _ := p.buf.Peek(bufferSize)
   307  	peekBuffer := bytes.NewBuffer(parserBufferPeekResult)
   308  
   309  	for {
   310  		peekData, peekErr := peekBuffer.ReadBytes('\n')
   311  		if peekErr != nil && peekErr != io.EOF {
   312  			p.debug("readPythonMultilines: failed to peek with error: %v", peekErr)
   313  			return "", peekErr
   314  		}
   315  
   316  		p.debug("readPythonMultilines: parsing %q", string(peekData))
   317  
   318  		peekMatches := pythonMultiline.FindStringSubmatch(string(peekData))
   319  		p.debug("readPythonMultilines: matched %d parts", len(peekMatches))
   320  		for n, v := range peekMatches {
   321  			p.debug("   %d: %q", n, v)
   322  		}
   323  
   324  		// Return if not a Python multiline value.
   325  		if len(peekMatches) != 3 {
   326  			p.debug("readPythonMultilines: end of value, got: %q", line)
   327  			return line, nil
   328  		}
   329  
   330  		// Advance the parser reader (buffer) in-sync with the peek buffer.
   331  		_, err := p.buf.Discard(len(peekData))
   332  		if err != nil {
   333  			p.debug("readPythonMultilines: failed to skip to the end, returning error")
   334  			return "", err
   335  		}
   336  
   337  		line += "\n" + peekMatches[0]
   338  	}
   339  }
   340  
   341  // parse parses data through an io.Reader.
   342  func (f *File) parse(reader io.Reader) (err error) {
   343  	p := newParser(reader, parserOptions{
   344  		IgnoreContinuation:          f.options.IgnoreContinuation,
   345  		IgnoreInlineComment:         f.options.IgnoreInlineComment,
   346  		AllowPythonMultilineValues:  f.options.AllowPythonMultilineValues,
   347  		SpaceBeforeInlineComment:    f.options.SpaceBeforeInlineComment,
   348  		UnescapeValueDoubleQuotes:   f.options.UnescapeValueDoubleQuotes,
   349  		UnescapeValueCommentSymbols: f.options.UnescapeValueCommentSymbols,
   350  		PreserveSurroundedQuote:     f.options.PreserveSurroundedQuote,
   351  		DebugFunc:                   f.options.DebugFunc,
   352  		ReaderBufferSize:            f.options.ReaderBufferSize,
   353  	})
   354  	if err = p.BOM(); err != nil {
   355  		return fmt.Errorf("BOM: %v", err)
   356  	}
   357  
   358  	// Ignore error because default section name is never empty string.
   359  	name := DefaultSection
   360  	if f.options.Insensitive || f.options.InsensitiveSections {
   361  		name = strings.ToLower(DefaultSection)
   362  	}
   363  	section, _ := f.NewSection(name)
   364  
   365  	// This "last" is not strictly equivalent to "previous one" if current key is not the first nested key
   366  	var isLastValueEmpty bool
   367  	var lastRegularKey *Key
   368  
   369  	var line []byte
   370  	var inUnparseableSection bool
   371  
   372  	// NOTE: Iterate and increase `currentPeekSize` until
   373  	// the size of the parser buffer is found.
   374  	// TODO(unknwon): When Golang 1.10 is the lowest version supported, replace with `parserBufferSize := p.buf.Size()`.
   375  	parserBufferSize := 0
   376  	// NOTE: Peek 4kb at a time.
   377  	currentPeekSize := minReaderBufferSize
   378  
   379  	if f.options.AllowPythonMultilineValues {
   380  		for {
   381  			peekBytes, _ := p.buf.Peek(currentPeekSize)
   382  			peekBytesLength := len(peekBytes)
   383  
   384  			if parserBufferSize >= peekBytesLength {
   385  				break
   386  			}
   387  
   388  			currentPeekSize *= 2
   389  			parserBufferSize = peekBytesLength
   390  		}
   391  	}
   392  
   393  	for !p.isEOF {
   394  		line, err = p.readUntil('\n')
   395  		if err != nil {
   396  			return err
   397  		}
   398  
   399  		if f.options.AllowNestedValues &&
   400  			isLastValueEmpty && len(line) > 0 {
   401  			if line[0] == ' ' || line[0] == '\t' {
   402  				err = lastRegularKey.addNestedValue(string(bytes.TrimSpace(line)))
   403  				if err != nil {
   404  					return err
   405  				}
   406  				continue
   407  			}
   408  		}
   409  
   410  		line = bytes.TrimLeftFunc(line, unicode.IsSpace)
   411  		if len(line) == 0 {
   412  			continue
   413  		}
   414  
   415  		// Comments
   416  		if line[0] == '#' || line[0] == ';' {
   417  			// Note: we do not care ending line break,
   418  			// it is needed for adding second line,
   419  			// so just clean it once at the end when set to value.
   420  			p.comment.Write(line)
   421  			continue
   422  		}
   423  
   424  		// Section
   425  		if line[0] == '[' {
   426  			// Read to the next ']' (TODO: support quoted strings)
   427  			closeIdx := bytes.LastIndexByte(line, ']')
   428  			if closeIdx == -1 {
   429  				return fmt.Errorf("unclosed section: %s", line)
   430  			}
   431  
   432  			name := string(line[1:closeIdx])
   433  			section, err = f.NewSection(name)
   434  			if err != nil {
   435  				return err
   436  			}
   437  
   438  			comment, has := cleanComment(line[closeIdx+1:])
   439  			if has {
   440  				p.comment.Write(comment)
   441  			}
   442  
   443  			section.Comment = strings.TrimSpace(p.comment.String())
   444  
   445  			// Reset auto-counter and comments
   446  			p.comment.Reset()
   447  			p.count = 1
   448  			// Nested values can't span sections
   449  			isLastValueEmpty = false
   450  
   451  			inUnparseableSection = false
   452  			for i := range f.options.UnparseableSections {
   453  				if f.options.UnparseableSections[i] == name ||
   454  					((f.options.Insensitive || f.options.InsensitiveSections) && strings.EqualFold(f.options.UnparseableSections[i], name)) {
   455  					inUnparseableSection = true
   456  					continue
   457  				}
   458  			}
   459  			continue
   460  		}
   461  
   462  		if inUnparseableSection {
   463  			section.isRawSection = true
   464  			section.rawBody += string(line)
   465  			continue
   466  		}
   467  
   468  		kname, offset, err := readKeyName(f.options.KeyValueDelimiters, line)
   469  		if err != nil {
   470  			switch {
   471  			// Treat as boolean key when desired, and whole line is key name.
   472  			case IsErrDelimiterNotFound(err):
   473  				switch {
   474  				case f.options.AllowBooleanKeys:
   475  					kname, err := p.readValue(line, parserBufferSize)
   476  					if err != nil {
   477  						return err
   478  					}
   479  					key, err := section.NewBooleanKey(kname)
   480  					if err != nil {
   481  						return err
   482  					}
   483  					key.Comment = strings.TrimSpace(p.comment.String())
   484  					p.comment.Reset()
   485  					continue
   486  
   487  				case f.options.SkipUnrecognizableLines:
   488  					continue
   489  				}
   490  			case IsErrEmptyKeyName(err) && f.options.SkipUnrecognizableLines:
   491  				continue
   492  			}
   493  			return err
   494  		}
   495  
   496  		// Auto increment.
   497  		isAutoIncr := false
   498  		if kname == "-" {
   499  			isAutoIncr = true
   500  			kname = "#" + strconv.Itoa(p.count)
   501  			p.count++
   502  		}
   503  
   504  		value, err := p.readValue(line[offset:], parserBufferSize)
   505  		if err != nil {
   506  			return err
   507  		}
   508  		isLastValueEmpty = len(value) == 0
   509  
   510  		key, err := section.NewKey(kname, value)
   511  		if err != nil {
   512  			return err
   513  		}
   514  		key.isAutoIncrement = isAutoIncr
   515  		key.Comment = strings.TrimSpace(p.comment.String())
   516  		p.comment.Reset()
   517  		lastRegularKey = key
   518  	}
   519  	return nil
   520  }
   521
View as plain text