byteio_reader.go

Documentation: sigs.k8s.io/kustomize/kyaml/kio

     1  // Copyright 2019 The Kubernetes Authors.
     2  // SPDX-License-Identifier: Apache-2.0
     3  
     4  package kio
     5  
     6  import (
     7  	"bytes"
     8  	"fmt"
     9  	"io"
    10  	"regexp"
    11  	"sort"
    12  	"strings"
    13  
    14  	"sigs.k8s.io/kustomize/kyaml/errors"
    15  	"sigs.k8s.io/kustomize/kyaml/kio/kioutil"
    16  	"sigs.k8s.io/kustomize/kyaml/yaml"
    17  )
    18  
    19  const (
    20  	ResourceListKind       = "ResourceList"
    21  	ResourceListAPIVersion = "config.kubernetes.io/v1"
    22  )
    23  
    24  // ByteReadWriter reads from an input and writes to an output.
    25  type ByteReadWriter struct {
    26  	// Reader is where ResourceNodes are decoded from.
    27  	Reader io.Reader
    28  
    29  	// Writer is where ResourceNodes are encoded.
    30  	Writer io.Writer
    31  
    32  	// OmitReaderAnnotations will configures Read to skip setting the config.kubernetes.io/index
    33  	// annotation on Resources as they are Read.
    34  	OmitReaderAnnotations bool
    35  
    36  	// KeepReaderAnnotations if set will keep the Reader specific annotations when writing
    37  	// the Resources, otherwise they will be cleared.
    38  	KeepReaderAnnotations bool
    39  
    40  	// PreserveSeqIndent if true adds kioutil.SeqIndentAnnotation to each resource
    41  	PreserveSeqIndent bool
    42  
    43  	// Style is a style that is set on the Resource Node Document.
    44  	Style yaml.Style
    45  
    46  	// WrapBareSeqNode wraps the bare sequence node document with map node,
    47  	// kyaml uses reader annotations to track resources, it is not possible to
    48  	// add them to bare sequence nodes, this option enables wrapping such bare
    49  	// sequence nodes into map node with key yaml.BareSeqNodeWrappingKey
    50  	// note that this wrapping is different and not related to ResourceList wrapping
    51  	WrapBareSeqNode bool
    52  
    53  	FunctionConfig *yaml.RNode
    54  
    55  	Results *yaml.RNode
    56  
    57  	NoWrap             bool
    58  	WrappingAPIVersion string
    59  	WrappingKind       string
    60  }
    61  
    62  func (rw *ByteReadWriter) Read() ([]*yaml.RNode, error) {
    63  	b := &ByteReader{
    64  		Reader:                rw.Reader,
    65  		OmitReaderAnnotations: rw.OmitReaderAnnotations,
    66  		PreserveSeqIndent:     rw.PreserveSeqIndent,
    67  		WrapBareSeqNode:       rw.WrapBareSeqNode,
    68  	}
    69  	val, err := b.Read()
    70  	rw.Results = b.Results
    71  
    72  	if rw.FunctionConfig == nil {
    73  		rw.FunctionConfig = b.FunctionConfig
    74  	}
    75  	if !rw.NoWrap && rw.WrappingKind == "" {
    76  		rw.WrappingAPIVersion = b.WrappingAPIVersion
    77  		rw.WrappingKind = b.WrappingKind
    78  	}
    79  	return val, errors.Wrap(err)
    80  }
    81  
    82  func (rw *ByteReadWriter) Write(nodes []*yaml.RNode) error {
    83  	w := ByteWriter{
    84  		Writer:                rw.Writer,
    85  		KeepReaderAnnotations: rw.KeepReaderAnnotations,
    86  		Style:                 rw.Style,
    87  		FunctionConfig:        rw.FunctionConfig,
    88  		Results:               rw.Results,
    89  	}
    90  	if !rw.NoWrap {
    91  		w.WrappingAPIVersion = rw.WrappingAPIVersion
    92  		w.WrappingKind = rw.WrappingKind
    93  	}
    94  	return w.Write(nodes)
    95  }
    96  
    97  // ParseAll reads all of the inputs into resources
    98  func ParseAll(inputs ...string) ([]*yaml.RNode, error) {
    99  	return (&ByteReader{
   100  		Reader: bytes.NewBufferString(strings.Join(inputs, "\n---\n")),
   101  	}).Read()
   102  }
   103  
   104  // FromBytes reads from a byte slice.
   105  func FromBytes(bs []byte) ([]*yaml.RNode, error) {
   106  	return (&ByteReader{
   107  		OmitReaderAnnotations: true,
   108  		AnchorsAweigh:         true,
   109  		Reader:                bytes.NewBuffer(bs),
   110  	}).Read()
   111  }
   112  
   113  // StringAll writes all of the resources to a string
   114  func StringAll(resources []*yaml.RNode) (string, error) {
   115  	var b bytes.Buffer
   116  	err := (&ByteWriter{Writer: &b}).Write(resources)
   117  	return b.String(), err
   118  }
   119  
   120  // ByteReader decodes ResourceNodes from bytes.
   121  // By default, Read will set the config.kubernetes.io/index annotation on each RNode as it
   122  // is read so they can be written back in the same order.
   123  type ByteReader struct {
   124  	// Reader is where ResourceNodes are decoded from.
   125  	Reader io.Reader
   126  
   127  	// OmitReaderAnnotations will configures Read to skip setting the config.kubernetes.io/index
   128  	// and internal.config.kubernetes.io/seqindent annotations on Resources as they are Read.
   129  	OmitReaderAnnotations bool
   130  
   131  	// PreserveSeqIndent if true adds kioutil.SeqIndentAnnotation to each resource
   132  	PreserveSeqIndent bool
   133  
   134  	// SetAnnotations is a map of caller specified annotations to set on resources as they are read
   135  	// These are independent of the annotations controlled by OmitReaderAnnotations
   136  	SetAnnotations map[string]string
   137  
   138  	FunctionConfig *yaml.RNode
   139  
   140  	Results *yaml.RNode
   141  
   142  	// DisableUnwrapping prevents Resources in Lists and ResourceLists from being unwrapped
   143  	DisableUnwrapping bool
   144  
   145  	// WrappingAPIVersion is set by Read(), and is the apiVersion of the object that
   146  	// the read objects were originally wrapped in.
   147  	WrappingAPIVersion string
   148  
   149  	// WrappingKind is set by Read(), and is the kind of the object that
   150  	// the read objects were originally wrapped in.
   151  	WrappingKind string
   152  
   153  	// WrapBareSeqNode wraps the bare sequence node document with map node,
   154  	// kyaml uses reader annotations to track resources, it is not possible to
   155  	// add them to bare sequence nodes, this option enables wrapping such bare
   156  	// sequence nodes into map node with key yaml.BareSeqNodeWrappingKey
   157  	// note that this wrapping is different and not related to ResourceList wrapping
   158  	WrapBareSeqNode bool
   159  
   160  	// AnchorsAweigh set to true attempts to replace all YAML anchor aliases
   161  	// with their definitions (anchor values) immediately after the read.
   162  	AnchorsAweigh bool
   163  }
   164  
   165  var _ Reader = &ByteReader{}
   166  
   167  // splitDocuments returns a slice of all documents contained in a YAML string. Multiple documents can be divided by the
   168  // YAML document separator (---). It allows for white space and comments to be after the separator on the same line,
   169  // but will return an error if anything else is on the line.
   170  func splitDocuments(s string) ([]string, error) {
   171  	docs := make([]string, 0)
   172  	if len(s) > 0 {
   173  		// The YAML document separator is any line that starts with ---
   174  		yamlSeparatorRegexp := regexp.MustCompile(`\n---.*\n`)
   175  
   176  		// Find all separators, check them for invalid content, and append each document to docs
   177  		separatorLocations := yamlSeparatorRegexp.FindAllStringIndex(s, -1)
   178  		prev := 0
   179  		for i := range separatorLocations {
   180  			loc := separatorLocations[i]
   181  			separator := s[loc[0]:loc[1]]
   182  
   183  			// If the next non-whitespace character on the line following the separator is not a comment, return an error
   184  			trimmedContentAfterSeparator := strings.TrimSpace(separator[4:])
   185  			if len(trimmedContentAfterSeparator) > 0 && trimmedContentAfterSeparator[0] != '#' {
   186  				return nil, errors.Errorf("invalid document separator: %s", strings.TrimSpace(separator))
   187  			}
   188  
   189  			docs = append(docs, s[prev:loc[0]])
   190  			prev = loc[1]
   191  		}
   192  		docs = append(docs, s[prev:])
   193  	}
   194  
   195  	return docs, nil
   196  }
   197  
   198  func (r *ByteReader) Read() ([]*yaml.RNode, error) {
   199  	if r.PreserveSeqIndent && r.OmitReaderAnnotations {
   200  		return nil, errors.Errorf(`"PreserveSeqIndent" option adds a reader annotation, please set "OmitReaderAnnotations" to false`)
   201  	}
   202  
   203  	output := ResourceNodeSlice{}
   204  
   205  	// by manually splitting resources -- otherwise the decoder will get the Resource
   206  	// boundaries wrong for header comments.
   207  	input := &bytes.Buffer{}
   208  	_, err := io.Copy(input, r.Reader)
   209  	if err != nil {
   210  		return nil, errors.Wrap(err)
   211  	}
   212  
   213  	// Replace the ending \r\n (line ending used in windows) with \n and then split it into multiple YAML documents
   214  	// if it contains document separators (---)
   215  	values, err := splitDocuments(strings.ReplaceAll(input.String(), "\r\n", "\n"))
   216  	if err != nil {
   217  		return nil, errors.Wrap(err)
   218  	}
   219  
   220  	index := 0
   221  	for i := range values {
   222  		// the Split used above will eat the tail '\n' from each resource. This may affect the
   223  		// literal string value since '\n' is meaningful in it.
   224  		if i != len(values)-1 {
   225  			values[i] += "\n"
   226  		}
   227  		decoder := yaml.NewDecoder(bytes.NewBufferString(values[i]))
   228  		node, err := r.decode(values[i], index, decoder)
   229  		if err == io.EOF {
   230  			continue
   231  		}
   232  
   233  		if err != nil {
   234  			return nil, errors.Wrap(err)
   235  		}
   236  		if yaml.IsMissingOrNull(node) {
   237  			// empty value
   238  			continue
   239  		}
   240  
   241  		// ok if no metadata -- assume not an InputList
   242  		meta, err := node.GetMeta()
   243  		if err != yaml.ErrMissingMetadata && err != nil {
   244  			return nil, errors.WrapPrefixf(err, "[%d]", i)
   245  		}
   246  
   247  		// the elements are wrapped in an InputList, unwrap them
   248  		// don't check apiVersion, we haven't standardized on the domain
   249  		if !r.DisableUnwrapping &&
   250  			len(values) == 1 && // Only unwrap if there is only 1 value
   251  			(meta.Kind == ResourceListKind || meta.Kind == "List") &&
   252  			(node.Field("items") != nil || node.Field("functionConfig") != nil) {
   253  			r.WrappingKind = meta.Kind
   254  			r.WrappingAPIVersion = meta.APIVersion
   255  
   256  			// unwrap the list
   257  			if fc := node.Field("functionConfig"); fc != nil {
   258  				r.FunctionConfig = fc.Value
   259  			}
   260  			if res := node.Field("results"); res != nil {
   261  				r.Results = res.Value
   262  			}
   263  
   264  			items := node.Field("items")
   265  			if items != nil {
   266  				for i := range items.Value.Content() {
   267  					// add items
   268  					output = append(output, yaml.NewRNode(items.Value.Content()[i]))
   269  				}
   270  			}
   271  			continue
   272  		}
   273  
   274  		// add the node to the list
   275  		output = append(output, node)
   276  
   277  		// increment the index annotation value
   278  		index++
   279  	}
   280  	if r.AnchorsAweigh {
   281  		for _, n := range output {
   282  			if err = n.DeAnchor(); err != nil {
   283  				return nil, err
   284  			}
   285  		}
   286  	}
   287  	return output, nil
   288  }
   289  
   290  func (r *ByteReader) decode(originalYAML string, index int, decoder *yaml.Decoder) (*yaml.RNode, error) {
   291  	node := &yaml.Node{}
   292  	err := decoder.Decode(node)
   293  	if err == io.EOF {
   294  		return nil, io.EOF
   295  	}
   296  	if err != nil {
   297  		return nil, errors.WrapPrefixf(err, "MalformedYAMLError")
   298  	}
   299  
   300  	if yaml.IsYNodeEmptyDoc(node) {
   301  		return nil, nil
   302  	}
   303  
   304  	// set annotations on the read Resources
   305  	// sort the annotations by key so the output Resources is consistent (otherwise the
   306  	// annotations will be in a random order)
   307  	n := yaml.NewRNode(node)
   308  	// check if it is a bare sequence node and wrap it with a yaml.BareSeqNodeWrappingKey
   309  	if r.WrapBareSeqNode && node.Kind == yaml.DocumentNode && len(node.Content) > 0 &&
   310  		node.Content[0] != nil && node.Content[0].Kind == yaml.SequenceNode {
   311  		wrappedNode := yaml.NewRNode(&yaml.Node{
   312  			Kind: yaml.MappingNode,
   313  		})
   314  		wrappedNode.PipeE(yaml.SetField(yaml.BareSeqNodeWrappingKey, n))
   315  		n = wrappedNode
   316  	}
   317  
   318  	if r.SetAnnotations == nil {
   319  		r.SetAnnotations = map[string]string{}
   320  	}
   321  	if !r.OmitReaderAnnotations {
   322  		err := kioutil.CopyLegacyAnnotations(n)
   323  		if err != nil {
   324  			return nil, err
   325  		}
   326  		r.SetAnnotations[kioutil.IndexAnnotation] = fmt.Sprintf("%d", index)
   327  		r.SetAnnotations[kioutil.LegacyIndexAnnotation] = fmt.Sprintf("%d", index)
   328  
   329  		if r.PreserveSeqIndent {
   330  			// derive and add the seqindent annotation
   331  			seqIndentStyle := yaml.DeriveSeqIndentStyle(originalYAML)
   332  			if seqIndentStyle != "" {
   333  				r.SetAnnotations[kioutil.SeqIndentAnnotation] = seqIndentStyle
   334  			}
   335  		}
   336  	}
   337  	var keys []string
   338  	for k := range r.SetAnnotations {
   339  		keys = append(keys, k)
   340  	}
   341  	sort.Strings(keys)
   342  	for _, k := range keys {
   343  		_, err = n.Pipe(yaml.SetAnnotation(k, r.SetAnnotations[k]))
   344  		if err != nil {
   345  			return nil, errors.Wrap(err)
   346  		}
   347  	}
   348  	return n, nil
   349  }
   350
View as plain text