encode.go

Documentation: google.golang.org/protobuf/proto

     1  // Copyright 2019 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package proto
     6  
     7  import (
     8  	"errors"
     9  	"fmt"
    10  
    11  	"google.golang.org/protobuf/encoding/protowire"
    12  	"google.golang.org/protobuf/internal/encoding/messageset"
    13  	"google.golang.org/protobuf/internal/order"
    14  	"google.golang.org/protobuf/internal/pragma"
    15  	"google.golang.org/protobuf/reflect/protoreflect"
    16  	"google.golang.org/protobuf/runtime/protoiface"
    17  
    18  	protoerrors "google.golang.org/protobuf/internal/errors"
    19  )
    20  
    21  // MarshalOptions configures the marshaler.
    22  //
    23  // Example usage:
    24  //
    25  //	b, err := MarshalOptions{Deterministic: true}.Marshal(m)
    26  type MarshalOptions struct {
    27  	pragma.NoUnkeyedLiterals
    28  
    29  	// AllowPartial allows messages that have missing required fields to marshal
    30  	// without returning an error. If AllowPartial is false (the default),
    31  	// Marshal will return an error if there are any missing required fields.
    32  	AllowPartial bool
    33  
    34  	// Deterministic controls whether the same message will always be
    35  	// serialized to the same bytes within the same binary.
    36  	//
    37  	// Setting this option guarantees that repeated serialization of
    38  	// the same message will return the same bytes, and that different
    39  	// processes of the same binary (which may be executing on different
    40  	// machines) will serialize equal messages to the same bytes.
    41  	// It has no effect on the resulting size of the encoded message compared
    42  	// to a non-deterministic marshal.
    43  	//
    44  	// Note that the deterministic serialization is NOT canonical across
    45  	// languages. It is not guaranteed to remain stable over time. It is
    46  	// unstable across different builds with schema changes due to unknown
    47  	// fields. Users who need canonical serialization (e.g., persistent
    48  	// storage in a canonical form, fingerprinting, etc.) must define
    49  	// their own canonicalization specification and implement their own
    50  	// serializer rather than relying on this API.
    51  	//
    52  	// If deterministic serialization is requested, map entries will be
    53  	// sorted by keys in lexographical order. This is an implementation
    54  	// detail and subject to change.
    55  	Deterministic bool
    56  
    57  	// UseCachedSize indicates that the result of a previous Size call
    58  	// may be reused.
    59  	//
    60  	// Setting this option asserts that:
    61  	//
    62  	// 1. Size has previously been called on this message with identical
    63  	// options (except for UseCachedSize itself).
    64  	//
    65  	// 2. The message and all its submessages have not changed in any
    66  	// way since the Size call.
    67  	//
    68  	// If either of these invariants is violated,
    69  	// the results are undefined and may include panics or corrupted output.
    70  	//
    71  	// Implementations MAY take this option into account to provide
    72  	// better performance, but there is no guarantee that they will do so.
    73  	// There is absolutely no guarantee that Size followed by Marshal with
    74  	// UseCachedSize set will perform equivalently to Marshal alone.
    75  	UseCachedSize bool
    76  }
    77  
    78  // flags turns the specified MarshalOptions (user-facing) into
    79  // protoiface.MarshalInputFlags (used internally by the marshaler).
    80  //
    81  // See impl.marshalOptions.Options for the inverse operation.
    82  func (o MarshalOptions) flags() protoiface.MarshalInputFlags {
    83  	var flags protoiface.MarshalInputFlags
    84  
    85  	// Note: o.AllowPartial is always forced to true by MarshalOptions.marshal,
    86  	// which is why it is not a part of MarshalInputFlags.
    87  
    88  	if o.Deterministic {
    89  		flags |= protoiface.MarshalDeterministic
    90  	}
    91  
    92  	if o.UseCachedSize {
    93  		flags |= protoiface.MarshalUseCachedSize
    94  	}
    95  
    96  	return flags
    97  }
    98  
    99  // Marshal returns the wire-format encoding of m.
   100  //
   101  // This is the most common entry point for encoding a Protobuf message.
   102  //
   103  // See the [MarshalOptions] type if you need more control.
   104  func Marshal(m Message) ([]byte, error) {
   105  	// Treat nil message interface as an empty message; nothing to output.
   106  	if m == nil {
   107  		return nil, nil
   108  	}
   109  
   110  	out, err := MarshalOptions{}.marshal(nil, m.ProtoReflect())
   111  	if len(out.Buf) == 0 && err == nil {
   112  		out.Buf = emptyBytesForMessage(m)
   113  	}
   114  	return out.Buf, err
   115  }
   116  
   117  // Marshal returns the wire-format encoding of m.
   118  func (o MarshalOptions) Marshal(m Message) ([]byte, error) {
   119  	// Treat nil message interface as an empty message; nothing to output.
   120  	if m == nil {
   121  		return nil, nil
   122  	}
   123  
   124  	out, err := o.marshal(nil, m.ProtoReflect())
   125  	if len(out.Buf) == 0 && err == nil {
   126  		out.Buf = emptyBytesForMessage(m)
   127  	}
   128  	return out.Buf, err
   129  }
   130  
   131  // emptyBytesForMessage returns a nil buffer if and only if m is invalid,
   132  // otherwise it returns a non-nil empty buffer.
   133  //
   134  // This is to assist the edge-case where user-code does the following:
   135  //
   136  //	m1.OptionalBytes, _ = proto.Marshal(m2)
   137  //
   138  // where they expect the proto2 "optional_bytes" field to be populated
   139  // if any only if m2 is a valid message.
   140  func emptyBytesForMessage(m Message) []byte {
   141  	if m == nil || !m.ProtoReflect().IsValid() {
   142  		return nil
   143  	}
   144  	return emptyBuf[:]
   145  }
   146  
   147  // MarshalAppend appends the wire-format encoding of m to b,
   148  // returning the result.
   149  //
   150  // This is a less common entry point than [Marshal], which is only needed if you
   151  // need to supply your own buffers for performance reasons.
   152  func (o MarshalOptions) MarshalAppend(b []byte, m Message) ([]byte, error) {
   153  	// Treat nil message interface as an empty message; nothing to append.
   154  	if m == nil {
   155  		return b, nil
   156  	}
   157  
   158  	out, err := o.marshal(b, m.ProtoReflect())
   159  	return out.Buf, err
   160  }
   161  
   162  // MarshalState returns the wire-format encoding of a message.
   163  //
   164  // This method permits fine-grained control over the marshaler.
   165  // Most users should use [Marshal] instead.
   166  func (o MarshalOptions) MarshalState(in protoiface.MarshalInput) (protoiface.MarshalOutput, error) {
   167  	return o.marshal(in.Buf, in.Message)
   168  }
   169  
   170  // marshal is a centralized function that all marshal operations go through.
   171  // For profiling purposes, avoid changing the name of this function or
   172  // introducing other code paths for marshal that do not go through this.
   173  func (o MarshalOptions) marshal(b []byte, m protoreflect.Message) (out protoiface.MarshalOutput, err error) {
   174  	allowPartial := o.AllowPartial
   175  	o.AllowPartial = true
   176  	if methods := protoMethods(m); methods != nil && methods.Marshal != nil &&
   177  		!(o.Deterministic && methods.Flags&protoiface.SupportMarshalDeterministic == 0) {
   178  		in := protoiface.MarshalInput{
   179  			Message: m,
   180  			Buf:     b,
   181  			Flags:   o.flags(),
   182  		}
   183  		if methods.Size != nil {
   184  			sout := methods.Size(protoiface.SizeInput{
   185  				Message: m,
   186  				Flags:   in.Flags,
   187  			})
   188  			if cap(b) < len(b)+sout.Size {
   189  				in.Buf = make([]byte, len(b), growcap(cap(b), len(b)+sout.Size))
   190  				copy(in.Buf, b)
   191  			}
   192  			in.Flags |= protoiface.MarshalUseCachedSize
   193  		}
   194  		out, err = methods.Marshal(in)
   195  	} else {
   196  		out.Buf, err = o.marshalMessageSlow(b, m)
   197  	}
   198  	if err != nil {
   199  		var mismatch *protoerrors.SizeMismatchError
   200  		if errors.As(err, &mismatch) {
   201  			return out, fmt.Errorf("marshaling %s: %v", string(m.Descriptor().FullName()), err)
   202  		}
   203  		return out, err
   204  	}
   205  	if allowPartial {
   206  		return out, nil
   207  	}
   208  	return out, checkInitialized(m)
   209  }
   210  
   211  func (o MarshalOptions) marshalMessage(b []byte, m protoreflect.Message) ([]byte, error) {
   212  	out, err := o.marshal(b, m)
   213  	return out.Buf, err
   214  }
   215  
   216  // growcap scales up the capacity of a slice.
   217  //
   218  // Given a slice with a current capacity of oldcap and a desired
   219  // capacity of wantcap, growcap returns a new capacity >= wantcap.
   220  //
   221  // The algorithm is mostly identical to the one used by append as of Go 1.14.
   222  func growcap(oldcap, wantcap int) (newcap int) {
   223  	if wantcap > oldcap*2 {
   224  		newcap = wantcap
   225  	} else if oldcap < 1024 {
   226  		// The Go 1.14 runtime takes this case when len(s) < 1024,
   227  		// not when cap(s) < 1024. The difference doesn't seem
   228  		// significant here.
   229  		newcap = oldcap * 2
   230  	} else {
   231  		newcap = oldcap
   232  		for 0 < newcap && newcap < wantcap {
   233  			newcap += newcap / 4
   234  		}
   235  		if newcap <= 0 {
   236  			newcap = wantcap
   237  		}
   238  	}
   239  	return newcap
   240  }
   241  
   242  func (o MarshalOptions) marshalMessageSlow(b []byte, m protoreflect.Message) ([]byte, error) {
   243  	if messageset.IsMessageSet(m.Descriptor()) {
   244  		return o.marshalMessageSet(b, m)
   245  	}
   246  	fieldOrder := order.AnyFieldOrder
   247  	if o.Deterministic {
   248  		// TODO: This should use a more natural ordering like NumberFieldOrder,
   249  		// but doing so breaks golden tests that make invalid assumption about
   250  		// output stability of this implementation.
   251  		fieldOrder = order.LegacyFieldOrder
   252  	}
   253  	var err error
   254  	order.RangeFields(m, fieldOrder, func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
   255  		b, err = o.marshalField(b, fd, v)
   256  		return err == nil
   257  	})
   258  	if err != nil {
   259  		return b, err
   260  	}
   261  	b = append(b, m.GetUnknown()...)
   262  	return b, nil
   263  }
   264  
   265  func (o MarshalOptions) marshalField(b []byte, fd protoreflect.FieldDescriptor, value protoreflect.Value) ([]byte, error) {
   266  	switch {
   267  	case fd.IsList():
   268  		return o.marshalList(b, fd, value.List())
   269  	case fd.IsMap():
   270  		return o.marshalMap(b, fd, value.Map())
   271  	default:
   272  		b = protowire.AppendTag(b, fd.Number(), wireTypes[fd.Kind()])
   273  		return o.marshalSingular(b, fd, value)
   274  	}
   275  }
   276  
   277  func (o MarshalOptions) marshalList(b []byte, fd protoreflect.FieldDescriptor, list protoreflect.List) ([]byte, error) {
   278  	if fd.IsPacked() && list.Len() > 0 {
   279  		b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)
   280  		b, pos := appendSpeculativeLength(b)
   281  		for i, llen := 0, list.Len(); i < llen; i++ {
   282  			var err error
   283  			b, err = o.marshalSingular(b, fd, list.Get(i))
   284  			if err != nil {
   285  				return b, err
   286  			}
   287  		}
   288  		b = finishSpeculativeLength(b, pos)
   289  		return b, nil
   290  	}
   291  
   292  	kind := fd.Kind()
   293  	for i, llen := 0, list.Len(); i < llen; i++ {
   294  		var err error
   295  		b = protowire.AppendTag(b, fd.Number(), wireTypes[kind])
   296  		b, err = o.marshalSingular(b, fd, list.Get(i))
   297  		if err != nil {
   298  			return b, err
   299  		}
   300  	}
   301  	return b, nil
   302  }
   303  
   304  func (o MarshalOptions) marshalMap(b []byte, fd protoreflect.FieldDescriptor, mapv protoreflect.Map) ([]byte, error) {
   305  	keyf := fd.MapKey()
   306  	valf := fd.MapValue()
   307  	keyOrder := order.AnyKeyOrder
   308  	if o.Deterministic {
   309  		keyOrder = order.GenericKeyOrder
   310  	}
   311  	var err error
   312  	order.RangeEntries(mapv, keyOrder, func(key protoreflect.MapKey, value protoreflect.Value) bool {
   313  		b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)
   314  		var pos int
   315  		b, pos = appendSpeculativeLength(b)
   316  
   317  		b, err = o.marshalField(b, keyf, key.Value())
   318  		if err != nil {
   319  			return false
   320  		}
   321  		b, err = o.marshalField(b, valf, value)
   322  		if err != nil {
   323  			return false
   324  		}
   325  		b = finishSpeculativeLength(b, pos)
   326  		return true
   327  	})
   328  	return b, err
   329  }
   330  
   331  // When encoding length-prefixed fields, we speculatively set aside some number of bytes
   332  // for the length, encode the data, and then encode the length (shifting the data if necessary
   333  // to make room).
   334  const speculativeLength = 1
   335  
   336  func appendSpeculativeLength(b []byte) ([]byte, int) {
   337  	pos := len(b)
   338  	b = append(b, "\x00\x00\x00\x00"[:speculativeLength]...)
   339  	return b, pos
   340  }
   341  
   342  func finishSpeculativeLength(b []byte, pos int) []byte {
   343  	mlen := len(b) - pos - speculativeLength
   344  	msiz := protowire.SizeVarint(uint64(mlen))
   345  	if msiz != speculativeLength {
   346  		for i := 0; i < msiz-speculativeLength; i++ {
   347  			b = append(b, 0)
   348  		}
   349  		copy(b[pos+msiz:], b[pos+speculativeLength:])
   350  		b = b[:pos+msiz+mlen]
   351  	}
   352  	protowire.AppendVarint(b[:pos], uint64(mlen))
   353  	return b
   354  }
   355
View as plain text