decoder.go

Documentation: k8s.io/apimachinery/pkg/util/yaml

     1  /*
     2  Copyright 2014 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package yaml
    18  
    19  import (
    20  	"bufio"
    21  	"bytes"
    22  	"encoding/json"
    23  	"fmt"
    24  	"io"
    25  	"strings"
    26  	"unicode"
    27  
    28  	jsonutil "k8s.io/apimachinery/pkg/util/json"
    29  
    30  	"sigs.k8s.io/yaml"
    31  )
    32  
    33  // Unmarshal unmarshals the given data
    34  // If v is a *map[string]interface{}, *[]interface{}, or *interface{} numbers
    35  // are converted to int64 or float64
    36  func Unmarshal(data []byte, v interface{}) error {
    37  	preserveIntFloat := func(d *json.Decoder) *json.Decoder {
    38  		d.UseNumber()
    39  		return d
    40  	}
    41  	switch v := v.(type) {
    42  	case *map[string]interface{}:
    43  		if err := yaml.Unmarshal(data, v, preserveIntFloat); err != nil {
    44  			return err
    45  		}
    46  		return jsonutil.ConvertMapNumbers(*v, 0)
    47  	case *[]interface{}:
    48  		if err := yaml.Unmarshal(data, v, preserveIntFloat); err != nil {
    49  			return err
    50  		}
    51  		return jsonutil.ConvertSliceNumbers(*v, 0)
    52  	case *interface{}:
    53  		if err := yaml.Unmarshal(data, v, preserveIntFloat); err != nil {
    54  			return err
    55  		}
    56  		return jsonutil.ConvertInterfaceNumbers(v, 0)
    57  	default:
    58  		return yaml.Unmarshal(data, v)
    59  	}
    60  }
    61  
    62  // UnmarshalStrict unmarshals the given data
    63  // strictly (erroring when there are duplicate fields).
    64  func UnmarshalStrict(data []byte, v interface{}) error {
    65  	preserveIntFloat := func(d *json.Decoder) *json.Decoder {
    66  		d.UseNumber()
    67  		return d
    68  	}
    69  	switch v := v.(type) {
    70  	case *map[string]interface{}:
    71  		if err := yaml.UnmarshalStrict(data, v, preserveIntFloat); err != nil {
    72  			return err
    73  		}
    74  		return jsonutil.ConvertMapNumbers(*v, 0)
    75  	case *[]interface{}:
    76  		if err := yaml.UnmarshalStrict(data, v, preserveIntFloat); err != nil {
    77  			return err
    78  		}
    79  		return jsonutil.ConvertSliceNumbers(*v, 0)
    80  	case *interface{}:
    81  		if err := yaml.UnmarshalStrict(data, v, preserveIntFloat); err != nil {
    82  			return err
    83  		}
    84  		return jsonutil.ConvertInterfaceNumbers(v, 0)
    85  	default:
    86  		return yaml.UnmarshalStrict(data, v)
    87  	}
    88  }
    89  
    90  // ToJSON converts a single YAML document into a JSON document
    91  // or returns an error. If the document appears to be JSON the
    92  // YAML decoding path is not used (so that error messages are
    93  // JSON specific).
    94  func ToJSON(data []byte) ([]byte, error) {
    95  	if hasJSONPrefix(data) {
    96  		return data, nil
    97  	}
    98  	return yaml.YAMLToJSON(data)
    99  }
   100  
   101  // YAMLToJSONDecoder decodes YAML documents from an io.Reader by
   102  // separating individual documents. It first converts the YAML
   103  // body to JSON, then unmarshals the JSON.
   104  type YAMLToJSONDecoder struct {
   105  	reader Reader
   106  }
   107  
   108  // NewYAMLToJSONDecoder decodes YAML documents from the provided
   109  // stream in chunks by converting each document (as defined by
   110  // the YAML spec) into its own chunk, converting it to JSON via
   111  // yaml.YAMLToJSON, and then passing it to json.Decoder.
   112  func NewYAMLToJSONDecoder(r io.Reader) *YAMLToJSONDecoder {
   113  	reader := bufio.NewReader(r)
   114  	return &YAMLToJSONDecoder{
   115  		reader: NewYAMLReader(reader),
   116  	}
   117  }
   118  
   119  // Decode reads a YAML document as JSON from the stream or returns
   120  // an error. The decoding rules match json.Unmarshal, not
   121  // yaml.Unmarshal.
   122  func (d *YAMLToJSONDecoder) Decode(into interface{}) error {
   123  	bytes, err := d.reader.Read()
   124  	if err != nil && err != io.EOF {
   125  		return err
   126  	}
   127  
   128  	if len(bytes) != 0 {
   129  		err := yaml.Unmarshal(bytes, into)
   130  		if err != nil {
   131  			return YAMLSyntaxError{err}
   132  		}
   133  	}
   134  	return err
   135  }
   136  
   137  // YAMLDecoder reads chunks of objects and returns ErrShortBuffer if
   138  // the data is not sufficient.
   139  type YAMLDecoder struct {
   140  	r         io.ReadCloser
   141  	scanner   *bufio.Scanner
   142  	remaining []byte
   143  }
   144  
   145  // NewDocumentDecoder decodes YAML documents from the provided
   146  // stream in chunks by converting each document (as defined by
   147  // the YAML spec) into its own chunk. io.ErrShortBuffer will be
   148  // returned if the entire buffer could not be read to assist
   149  // the caller in framing the chunk.
   150  func NewDocumentDecoder(r io.ReadCloser) io.ReadCloser {
   151  	scanner := bufio.NewScanner(r)
   152  	// the size of initial allocation for buffer 4k
   153  	buf := make([]byte, 4*1024)
   154  	// the maximum size used to buffer a token 5M
   155  	scanner.Buffer(buf, 5*1024*1024)
   156  	scanner.Split(splitYAMLDocument)
   157  	return &YAMLDecoder{
   158  		r:       r,
   159  		scanner: scanner,
   160  	}
   161  }
   162  
   163  // Read reads the previous slice into the buffer, or attempts to read
   164  // the next chunk.
   165  // TODO: switch to readline approach.
   166  func (d *YAMLDecoder) Read(data []byte) (n int, err error) {
   167  	left := len(d.remaining)
   168  	if left == 0 {
   169  		// return the next chunk from the stream
   170  		if !d.scanner.Scan() {
   171  			err := d.scanner.Err()
   172  			if err == nil {
   173  				err = io.EOF
   174  			}
   175  			return 0, err
   176  		}
   177  		out := d.scanner.Bytes()
   178  		d.remaining = out
   179  		left = len(out)
   180  	}
   181  
   182  	// fits within data
   183  	if left <= len(data) {
   184  		copy(data, d.remaining)
   185  		d.remaining = nil
   186  		return left, nil
   187  	}
   188  
   189  	// caller will need to reread
   190  	copy(data, d.remaining[:len(data)])
   191  	d.remaining = d.remaining[len(data):]
   192  	return len(data), io.ErrShortBuffer
   193  }
   194  
   195  func (d *YAMLDecoder) Close() error {
   196  	return d.r.Close()
   197  }
   198  
   199  const yamlSeparator = "\n---"
   200  const separator = "---"
   201  
   202  // splitYAMLDocument is a bufio.SplitFunc for splitting YAML streams into individual documents.
   203  func splitYAMLDocument(data []byte, atEOF bool) (advance int, token []byte, err error) {
   204  	if atEOF && len(data) == 0 {
   205  		return 0, nil, nil
   206  	}
   207  	sep := len([]byte(yamlSeparator))
   208  	if i := bytes.Index(data, []byte(yamlSeparator)); i >= 0 {
   209  		// We have a potential document terminator
   210  		i += sep
   211  		after := data[i:]
   212  		if len(after) == 0 {
   213  			// we can't read any more characters
   214  			if atEOF {
   215  				return len(data), data[:len(data)-sep], nil
   216  			}
   217  			return 0, nil, nil
   218  		}
   219  		if j := bytes.IndexByte(after, '\n'); j >= 0 {
   220  			return i + j + 1, data[0 : i-sep], nil
   221  		}
   222  		return 0, nil, nil
   223  	}
   224  	// If we're at EOF, we have a final, non-terminated line. Return it.
   225  	if atEOF {
   226  		return len(data), data, nil
   227  	}
   228  	// Request more data.
   229  	return 0, nil, nil
   230  }
   231  
   232  // decoder is a convenience interface for Decode.
   233  type decoder interface {
   234  	Decode(into interface{}) error
   235  }
   236  
   237  // YAMLOrJSONDecoder attempts to decode a stream of JSON documents or
   238  // YAML documents by sniffing for a leading { character.
   239  type YAMLOrJSONDecoder struct {
   240  	r          io.Reader
   241  	bufferSize int
   242  
   243  	decoder decoder
   244  }
   245  
   246  type JSONSyntaxError struct {
   247  	Offset int64
   248  	Err    error
   249  }
   250  
   251  func (e JSONSyntaxError) Error() string {
   252  	return fmt.Sprintf("json: offset %d: %s", e.Offset, e.Err.Error())
   253  }
   254  
   255  type YAMLSyntaxError struct {
   256  	err error
   257  }
   258  
   259  func (e YAMLSyntaxError) Error() string {
   260  	return e.err.Error()
   261  }
   262  
   263  // NewYAMLOrJSONDecoder returns a decoder that will process YAML documents
   264  // or JSON documents from the given reader as a stream. bufferSize determines
   265  // how far into the stream the decoder will look to figure out whether this
   266  // is a JSON stream (has whitespace followed by an open brace).
   267  func NewYAMLOrJSONDecoder(r io.Reader, bufferSize int) *YAMLOrJSONDecoder {
   268  	return &YAMLOrJSONDecoder{
   269  		r:          r,
   270  		bufferSize: bufferSize,
   271  	}
   272  }
   273  
   274  // Decode unmarshals the next object from the underlying stream into the
   275  // provide object, or returns an error.
   276  func (d *YAMLOrJSONDecoder) Decode(into interface{}) error {
   277  	if d.decoder == nil {
   278  		buffer, _, isJSON := GuessJSONStream(d.r, d.bufferSize)
   279  		if isJSON {
   280  			d.decoder = json.NewDecoder(buffer)
   281  		} else {
   282  			d.decoder = NewYAMLToJSONDecoder(buffer)
   283  		}
   284  	}
   285  	err := d.decoder.Decode(into)
   286  	if syntax, ok := err.(*json.SyntaxError); ok {
   287  		return JSONSyntaxError{
   288  			Offset: syntax.Offset,
   289  			Err:    syntax,
   290  		}
   291  	}
   292  	return err
   293  }
   294  
   295  type Reader interface {
   296  	Read() ([]byte, error)
   297  }
   298  
   299  type YAMLReader struct {
   300  	reader Reader
   301  }
   302  
   303  func NewYAMLReader(r *bufio.Reader) *YAMLReader {
   304  	return &YAMLReader{
   305  		reader: &LineReader{reader: r},
   306  	}
   307  }
   308  
   309  // Read returns a full YAML document.
   310  func (r *YAMLReader) Read() ([]byte, error) {
   311  	var buffer bytes.Buffer
   312  	for {
   313  		line, err := r.reader.Read()
   314  		if err != nil && err != io.EOF {
   315  			return nil, err
   316  		}
   317  
   318  		sep := len([]byte(separator))
   319  		if i := bytes.Index(line, []byte(separator)); i == 0 {
   320  			// We have a potential document terminator
   321  			i += sep
   322  			trimmed := strings.TrimSpace(string(line[i:]))
   323  			// We only allow comments and spaces following the yaml doc separator, otherwise we'll return an error
   324  			if len(trimmed) > 0 && string(trimmed[0]) != "#" {
   325  				return nil, YAMLSyntaxError{
   326  					err: fmt.Errorf("invalid Yaml document separator: %s", trimmed),
   327  				}
   328  			}
   329  			if buffer.Len() != 0 {
   330  				return buffer.Bytes(), nil
   331  			}
   332  			if err == io.EOF {
   333  				return nil, err
   334  			}
   335  		}
   336  		if err == io.EOF {
   337  			if buffer.Len() != 0 {
   338  				// If we're at EOF, we have a final, non-terminated line. Return it.
   339  				return buffer.Bytes(), nil
   340  			}
   341  			return nil, err
   342  		}
   343  		buffer.Write(line)
   344  	}
   345  }
   346  
   347  type LineReader struct {
   348  	reader *bufio.Reader
   349  }
   350  
   351  // Read returns a single line (with '\n' ended) from the underlying reader.
   352  // An error is returned iff there is an error with the underlying reader.
   353  func (r *LineReader) Read() ([]byte, error) {
   354  	var (
   355  		isPrefix bool  = true
   356  		err      error = nil
   357  		line     []byte
   358  		buffer   bytes.Buffer
   359  	)
   360  
   361  	for isPrefix && err == nil {
   362  		line, isPrefix, err = r.reader.ReadLine()
   363  		buffer.Write(line)
   364  	}
   365  	buffer.WriteByte('\n')
   366  	return buffer.Bytes(), err
   367  }
   368  
   369  // GuessJSONStream scans the provided reader up to size, looking
   370  // for an open brace indicating this is JSON. It will return the
   371  // bufio.Reader it creates for the consumer.
   372  func GuessJSONStream(r io.Reader, size int) (io.Reader, []byte, bool) {
   373  	buffer := bufio.NewReaderSize(r, size)
   374  	b, _ := buffer.Peek(size)
   375  	return buffer, b, hasJSONPrefix(b)
   376  }
   377  
   378  // IsJSONBuffer scans the provided buffer, looking
   379  // for an open brace indicating this is JSON.
   380  func IsJSONBuffer(buf []byte) bool {
   381  	return hasJSONPrefix(buf)
   382  }
   383  
   384  var jsonPrefix = []byte("{")
   385  
   386  // hasJSONPrefix returns true if the provided buffer appears to start with
   387  // a JSON open brace.
   388  func hasJSONPrefix(buf []byte) bool {
   389  	return hasPrefix(buf, jsonPrefix)
   390  }
   391  
   392  // Return true if the first non-whitespace bytes in buf is
   393  // prefix.
   394  func hasPrefix(buf []byte, prefix []byte) bool {
   395  	trim := bytes.TrimLeftFunc(buf, unicode.IsSpace)
   396  	return bytes.HasPrefix(trim, prefix)
   397  }
   398
View as plain text