readerc.go

Documentation: gopkg.in/yaml.v2

     1  package yaml
     2  
     3  import (
     4  	"io"
     5  )
     6  
     7  // Set the reader error and return 0.
     8  func yaml_parser_set_reader_error(parser *yaml_parser_t, problem string, offset int, value int) bool {
     9  	parser.error = yaml_READER_ERROR
    10  	parser.problem = problem
    11  	parser.problem_offset = offset
    12  	parser.problem_value = value
    13  	return false
    14  }
    15  
    16  // Byte order marks.
    17  const (
    18  	bom_UTF8    = "\xef\xbb\xbf"
    19  	bom_UTF16LE = "\xff\xfe"
    20  	bom_UTF16BE = "\xfe\xff"
    21  )
    22  
    23  // Determine the input stream encoding by checking the BOM symbol. If no BOM is
    24  // found, the UTF-8 encoding is assumed. Return 1 on success, 0 on failure.
    25  func yaml_parser_determine_encoding(parser *yaml_parser_t) bool {
    26  	// Ensure that we had enough bytes in the raw buffer.
    27  	for !parser.eof && len(parser.raw_buffer)-parser.raw_buffer_pos < 3 {
    28  		if !yaml_parser_update_raw_buffer(parser) {
    29  			return false
    30  		}
    31  	}
    32  
    33  	// Determine the encoding.
    34  	buf := parser.raw_buffer
    35  	pos := parser.raw_buffer_pos
    36  	avail := len(buf) - pos
    37  	if avail >= 2 && buf[pos] == bom_UTF16LE[0] && buf[pos+1] == bom_UTF16LE[1] {
    38  		parser.encoding = yaml_UTF16LE_ENCODING
    39  		parser.raw_buffer_pos += 2
    40  		parser.offset += 2
    41  	} else if avail >= 2 && buf[pos] == bom_UTF16BE[0] && buf[pos+1] == bom_UTF16BE[1] {
    42  		parser.encoding = yaml_UTF16BE_ENCODING
    43  		parser.raw_buffer_pos += 2
    44  		parser.offset += 2
    45  	} else if avail >= 3 && buf[pos] == bom_UTF8[0] && buf[pos+1] == bom_UTF8[1] && buf[pos+2] == bom_UTF8[2] {
    46  		parser.encoding = yaml_UTF8_ENCODING
    47  		parser.raw_buffer_pos += 3
    48  		parser.offset += 3
    49  	} else {
    50  		parser.encoding = yaml_UTF8_ENCODING
    51  	}
    52  	return true
    53  }
    54  
    55  // Update the raw buffer.
    56  func yaml_parser_update_raw_buffer(parser *yaml_parser_t) bool {
    57  	size_read := 0
    58  
    59  	// Return if the raw buffer is full.
    60  	if parser.raw_buffer_pos == 0 && len(parser.raw_buffer) == cap(parser.raw_buffer) {
    61  		return true
    62  	}
    63  
    64  	// Return on EOF.
    65  	if parser.eof {
    66  		return true
    67  	}
    68  
    69  	// Move the remaining bytes in the raw buffer to the beginning.
    70  	if parser.raw_buffer_pos > 0 && parser.raw_buffer_pos < len(parser.raw_buffer) {
    71  		copy(parser.raw_buffer, parser.raw_buffer[parser.raw_buffer_pos:])
    72  	}
    73  	parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)-parser.raw_buffer_pos]
    74  	parser.raw_buffer_pos = 0
    75  
    76  	// Call the read handler to fill the buffer.
    77  	size_read, err := parser.read_handler(parser, parser.raw_buffer[len(parser.raw_buffer):cap(parser.raw_buffer)])
    78  	parser.raw_buffer = parser.raw_buffer[:len(parser.raw_buffer)+size_read]
    79  	if err == io.EOF {
    80  		parser.eof = true
    81  	} else if err != nil {
    82  		return yaml_parser_set_reader_error(parser, "input error: "+err.Error(), parser.offset, -1)
    83  	}
    84  	return true
    85  }
    86  
    87  // Ensure that the buffer contains at least `length` characters.
    88  // Return true on success, false on failure.
    89  //
    90  // The length is supposed to be significantly less that the buffer size.
    91  func yaml_parser_update_buffer(parser *yaml_parser_t, length int) bool {
    92  	if parser.read_handler == nil {
    93  		panic("read handler must be set")
    94  	}
    95  
    96  	// [Go] This function was changed to guarantee the requested length size at EOF.
    97  	// The fact we need to do this is pretty awful, but the description above implies
    98  	// for that to be the case, and there are tests 
    99  
   100  	// If the EOF flag is set and the raw buffer is empty, do nothing.
   101  	if parser.eof && parser.raw_buffer_pos == len(parser.raw_buffer) {
   102  		// [Go] ACTUALLY! Read the documentation of this function above.
   103  		// This is just broken. To return true, we need to have the
   104  		// given length in the buffer. Not doing that means every single
   105  		// check that calls this function to make sure the buffer has a
   106  		// given length is Go) panicking; or C) accessing invalid memory.
   107  		//return true
   108  	}
   109  
   110  	// Return if the buffer contains enough characters.
   111  	if parser.unread >= length {
   112  		return true
   113  	}
   114  
   115  	// Determine the input encoding if it is not known yet.
   116  	if parser.encoding == yaml_ANY_ENCODING {
   117  		if !yaml_parser_determine_encoding(parser) {
   118  			return false
   119  		}
   120  	}
   121  
   122  	// Move the unread characters to the beginning of the buffer.
   123  	buffer_len := len(parser.buffer)
   124  	if parser.buffer_pos > 0 && parser.buffer_pos < buffer_len {
   125  		copy(parser.buffer, parser.buffer[parser.buffer_pos:])
   126  		buffer_len -= parser.buffer_pos
   127  		parser.buffer_pos = 0
   128  	} else if parser.buffer_pos == buffer_len {
   129  		buffer_len = 0
   130  		parser.buffer_pos = 0
   131  	}
   132  
   133  	// Open the whole buffer for writing, and cut it before returning.
   134  	parser.buffer = parser.buffer[:cap(parser.buffer)]
   135  
   136  	// Fill the buffer until it has enough characters.
   137  	first := true
   138  	for parser.unread < length {
   139  
   140  		// Fill the raw buffer if necessary.
   141  		if !first || parser.raw_buffer_pos == len(parser.raw_buffer) {
   142  			if !yaml_parser_update_raw_buffer(parser) {
   143  				parser.buffer = parser.buffer[:buffer_len]
   144  				return false
   145  			}
   146  		}
   147  		first = false
   148  
   149  		// Decode the raw buffer.
   150  	inner:
   151  		for parser.raw_buffer_pos != len(parser.raw_buffer) {
   152  			var value rune
   153  			var width int
   154  
   155  			raw_unread := len(parser.raw_buffer) - parser.raw_buffer_pos
   156  
   157  			// Decode the next character.
   158  			switch parser.encoding {
   159  			case yaml_UTF8_ENCODING:
   160  				// Decode a UTF-8 character.  Check RFC 3629
   161  				// (http://www.ietf.org/rfc/rfc3629.txt) for more details.
   162  				//
   163  				// The following table (taken from the RFC) is used for
   164  				// decoding.
   165  				//
   166  				//    Char. number range |        UTF-8 octet sequence
   167  				//      (hexadecimal)    |              (binary)
   168  				//   --------------------+------------------------------------
   169  				//   0000 0000-0000 007F | 0xxxxxxx
   170  				//   0000 0080-0000 07FF | 110xxxxx 10xxxxxx
   171  				//   0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
   172  				//   0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
   173  				//
   174  				// Additionally, the characters in the range 0xD800-0xDFFF
   175  				// are prohibited as they are reserved for use with UTF-16
   176  				// surrogate pairs.
   177  
   178  				// Determine the length of the UTF-8 sequence.
   179  				octet := parser.raw_buffer[parser.raw_buffer_pos]
   180  				switch {
   181  				case octet&0x80 == 0x00:
   182  					width = 1
   183  				case octet&0xE0 == 0xC0:
   184  					width = 2
   185  				case octet&0xF0 == 0xE0:
   186  					width = 3
   187  				case octet&0xF8 == 0xF0:
   188  					width = 4
   189  				default:
   190  					// The leading octet is invalid.
   191  					return yaml_parser_set_reader_error(parser,
   192  						"invalid leading UTF-8 octet",
   193  						parser.offset, int(octet))
   194  				}
   195  
   196  				// Check if the raw buffer contains an incomplete character.
   197  				if width > raw_unread {
   198  					if parser.eof {
   199  						return yaml_parser_set_reader_error(parser,
   200  							"incomplete UTF-8 octet sequence",
   201  							parser.offset, -1)
   202  					}
   203  					break inner
   204  				}
   205  
   206  				// Decode the leading octet.
   207  				switch {
   208  				case octet&0x80 == 0x00:
   209  					value = rune(octet & 0x7F)
   210  				case octet&0xE0 == 0xC0:
   211  					value = rune(octet & 0x1F)
   212  				case octet&0xF0 == 0xE0:
   213  					value = rune(octet & 0x0F)
   214  				case octet&0xF8 == 0xF0:
   215  					value = rune(octet & 0x07)
   216  				default:
   217  					value = 0
   218  				}
   219  
   220  				// Check and decode the trailing octets.
   221  				for k := 1; k < width; k++ {
   222  					octet = parser.raw_buffer[parser.raw_buffer_pos+k]
   223  
   224  					// Check if the octet is valid.
   225  					if (octet & 0xC0) != 0x80 {
   226  						return yaml_parser_set_reader_error(parser,
   227  							"invalid trailing UTF-8 octet",
   228  							parser.offset+k, int(octet))
   229  					}
   230  
   231  					// Decode the octet.
   232  					value = (value << 6) + rune(octet&0x3F)
   233  				}
   234  
   235  				// Check the length of the sequence against the value.
   236  				switch {
   237  				case width == 1:
   238  				case width == 2 && value >= 0x80:
   239  				case width == 3 && value >= 0x800:
   240  				case width == 4 && value >= 0x10000:
   241  				default:
   242  					return yaml_parser_set_reader_error(parser,
   243  						"invalid length of a UTF-8 sequence",
   244  						parser.offset, -1)
   245  				}
   246  
   247  				// Check the range of the value.
   248  				if value >= 0xD800 && value <= 0xDFFF || value > 0x10FFFF {
   249  					return yaml_parser_set_reader_error(parser,
   250  						"invalid Unicode character",
   251  						parser.offset, int(value))
   252  				}
   253  
   254  			case yaml_UTF16LE_ENCODING, yaml_UTF16BE_ENCODING:
   255  				var low, high int
   256  				if parser.encoding == yaml_UTF16LE_ENCODING {
   257  					low, high = 0, 1
   258  				} else {
   259  					low, high = 1, 0
   260  				}
   261  
   262  				// The UTF-16 encoding is not as simple as one might
   263  				// naively think.  Check RFC 2781
   264  				// (http://www.ietf.org/rfc/rfc2781.txt).
   265  				//
   266  				// Normally, two subsequent bytes describe a Unicode
   267  				// character.  However a special technique (called a
   268  				// surrogate pair) is used for specifying character
   269  				// values larger than 0xFFFF.
   270  				//
   271  				// A surrogate pair consists of two pseudo-characters:
   272  				//      high surrogate area (0xD800-0xDBFF)
   273  				//      low surrogate area (0xDC00-0xDFFF)
   274  				//
   275  				// The following formulas are used for decoding
   276  				// and encoding characters using surrogate pairs:
   277  				//
   278  				//  U  = U' + 0x10000   (0x01 00 00 <= U <= 0x10 FF FF)
   279  				//  U' = yyyyyyyyyyxxxxxxxxxx   (0 <= U' <= 0x0F FF FF)
   280  				//  W1 = 110110yyyyyyyyyy
   281  				//  W2 = 110111xxxxxxxxxx
   282  				//
   283  				// where U is the character value, W1 is the high surrogate
   284  				// area, W2 is the low surrogate area.
   285  
   286  				// Check for incomplete UTF-16 character.
   287  				if raw_unread < 2 {
   288  					if parser.eof {
   289  						return yaml_parser_set_reader_error(parser,
   290  							"incomplete UTF-16 character",
   291  							parser.offset, -1)
   292  					}
   293  					break inner
   294  				}
   295  
   296  				// Get the character.
   297  				value = rune(parser.raw_buffer[parser.raw_buffer_pos+low]) +
   298  					(rune(parser.raw_buffer[parser.raw_buffer_pos+high]) << 8)
   299  
   300  				// Check for unexpected low surrogate area.
   301  				if value&0xFC00 == 0xDC00 {
   302  					return yaml_parser_set_reader_error(parser,
   303  						"unexpected low surrogate area",
   304  						parser.offset, int(value))
   305  				}
   306  
   307  				// Check for a high surrogate area.
   308  				if value&0xFC00 == 0xD800 {
   309  					width = 4
   310  
   311  					// Check for incomplete surrogate pair.
   312  					if raw_unread < 4 {
   313  						if parser.eof {
   314  							return yaml_parser_set_reader_error(parser,
   315  								"incomplete UTF-16 surrogate pair",
   316  								parser.offset, -1)
   317  						}
   318  						break inner
   319  					}
   320  
   321  					// Get the next character.
   322  					value2 := rune(parser.raw_buffer[parser.raw_buffer_pos+low+2]) +
   323  						(rune(parser.raw_buffer[parser.raw_buffer_pos+high+2]) << 8)
   324  
   325  					// Check for a low surrogate area.
   326  					if value2&0xFC00 != 0xDC00 {
   327  						return yaml_parser_set_reader_error(parser,
   328  							"expected low surrogate area",
   329  							parser.offset+2, int(value2))
   330  					}
   331  
   332  					// Generate the value of the surrogate pair.
   333  					value = 0x10000 + ((value & 0x3FF) << 10) + (value2 & 0x3FF)
   334  				} else {
   335  					width = 2
   336  				}
   337  
   338  			default:
   339  				panic("impossible")
   340  			}
   341  
   342  			// Check if the character is in the allowed range:
   343  			//      #x9 | #xA | #xD | [#x20-#x7E]               (8 bit)
   344  			//      | #x85 | [#xA0-#xD7FF] | [#xE000-#xFFFD]    (16 bit)
   345  			//      | [#x10000-#x10FFFF]                        (32 bit)
   346  			switch {
   347  			case value == 0x09:
   348  			case value == 0x0A:
   349  			case value == 0x0D:
   350  			case value >= 0x20 && value <= 0x7E:
   351  			case value == 0x85:
   352  			case value >= 0xA0 && value <= 0xD7FF:
   353  			case value >= 0xE000 && value <= 0xFFFD:
   354  			case value >= 0x10000 && value <= 0x10FFFF:
   355  			default:
   356  				return yaml_parser_set_reader_error(parser,
   357  					"control characters are not allowed",
   358  					parser.offset, int(value))
   359  			}
   360  
   361  			// Move the raw pointers.
   362  			parser.raw_buffer_pos += width
   363  			parser.offset += width
   364  
   365  			// Finally put the character into the buffer.
   366  			if value <= 0x7F {
   367  				// 0000 0000-0000 007F . 0xxxxxxx
   368  				parser.buffer[buffer_len+0] = byte(value)
   369  				buffer_len += 1
   370  			} else if value <= 0x7FF {
   371  				// 0000 0080-0000 07FF . 110xxxxx 10xxxxxx
   372  				parser.buffer[buffer_len+0] = byte(0xC0 + (value >> 6))
   373  				parser.buffer[buffer_len+1] = byte(0x80 + (value & 0x3F))
   374  				buffer_len += 2
   375  			} else if value <= 0xFFFF {
   376  				// 0000 0800-0000 FFFF . 1110xxxx 10xxxxxx 10xxxxxx
   377  				parser.buffer[buffer_len+0] = byte(0xE0 + (value >> 12))
   378  				parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 6) & 0x3F))
   379  				parser.buffer[buffer_len+2] = byte(0x80 + (value & 0x3F))
   380  				buffer_len += 3
   381  			} else {
   382  				// 0001 0000-0010 FFFF . 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
   383  				parser.buffer[buffer_len+0] = byte(0xF0 + (value >> 18))
   384  				parser.buffer[buffer_len+1] = byte(0x80 + ((value >> 12) & 0x3F))
   385  				parser.buffer[buffer_len+2] = byte(0x80 + ((value >> 6) & 0x3F))
   386  				parser.buffer[buffer_len+3] = byte(0x80 + (value & 0x3F))
   387  				buffer_len += 4
   388  			}
   389  
   390  			parser.unread++
   391  		}
   392  
   393  		// On EOF, put NUL into the buffer and return.
   394  		if parser.eof {
   395  			parser.buffer[buffer_len] = 0
   396  			buffer_len++
   397  			parser.unread++
   398  			break
   399  		}
   400  	}
   401  	// [Go] Read the documentation of this function above. To return true,
   402  	// we need to have the given length in the buffer. Not doing that means
   403  	// every single check that calls this function to make sure the buffer
   404  	// has a given length is Go) panicking; or C) accessing invalid memory.
   405  	// This happens here due to the EOF above breaking early.
   406  	for buffer_len < length {
   407  		parser.buffer[buffer_len] = 0
   408  		buffer_len++
   409  	}
   410  	parser.buffer = parser.buffer[:buffer_len]
   411  	return true
   412  }
   413
View as plain text