Source file src/github.com/rs/zerolog/internal/cbor/decode_stream.go

Documentation: github.com/rs/zerolog/internal/cbor

     1  package cbor
     2  
     3  // This file contains code to decode a stream of CBOR Data into JSON.
     4  
     5  import (
     6  	"bufio"
     7  	"bytes"
     8  	"fmt"
     9  	"io"
    10  	"math"
    11  	"net"
    12  	"runtime"
    13  	"strconv"
    14  	"strings"
    15  	"time"
    16  	"unicode/utf8"
    17  )
    18  
    19  var decodeTimeZone *time.Location
    20  
    21  const hexTable = "0123456789abcdef"
    22  
    23  const isFloat32 = 4
    24  const isFloat64 = 8
    25  
    26  func readNBytes(src *bufio.Reader, n int) []byte {
    27  	ret := make([]byte, n)
    28  	for i := 0; i < n; i++ {
    29  		ch, e := src.ReadByte()
    30  		if e != nil {
    31  			panic(fmt.Errorf("Tried to Read %d Bytes.. But hit end of file", n))
    32  		}
    33  		ret[i] = ch
    34  	}
    35  	return ret
    36  }
    37  
    38  func readByte(src *bufio.Reader) byte {
    39  	b, e := src.ReadByte()
    40  	if e != nil {
    41  		panic(fmt.Errorf("Tried to Read 1 Byte.. But hit end of file"))
    42  	}
    43  	return b
    44  }
    45  
    46  func decodeIntAdditionalType(src *bufio.Reader, minor byte) int64 {
    47  	val := int64(0)
    48  	if minor <= 23 {
    49  		val = int64(minor)
    50  	} else {
    51  		bytesToRead := 0
    52  		switch minor {
    53  		case additionalTypeIntUint8:
    54  			bytesToRead = 1
    55  		case additionalTypeIntUint16:
    56  			bytesToRead = 2
    57  		case additionalTypeIntUint32:
    58  			bytesToRead = 4
    59  		case additionalTypeIntUint64:
    60  			bytesToRead = 8
    61  		default:
    62  			panic(fmt.Errorf("Invalid Additional Type: %d in decodeInteger (expected <28)", minor))
    63  		}
    64  		pb := readNBytes(src, bytesToRead)
    65  		for i := 0; i < bytesToRead; i++ {
    66  			val = val * 256
    67  			val += int64(pb[i])
    68  		}
    69  	}
    70  	return val
    71  }
    72  
    73  func decodeInteger(src *bufio.Reader) int64 {
    74  	pb := readByte(src)
    75  	major := pb & maskOutAdditionalType
    76  	minor := pb & maskOutMajorType
    77  	if major != majorTypeUnsignedInt && major != majorTypeNegativeInt {
    78  		panic(fmt.Errorf("Major type is: %d in decodeInteger!! (expected 0 or 1)", major))
    79  	}
    80  	val := decodeIntAdditionalType(src, minor)
    81  	if major == 0 {
    82  		return val
    83  	}
    84  	return (-1 - val)
    85  }
    86  
    87  func decodeFloat(src *bufio.Reader) (float64, int) {
    88  	pb := readByte(src)
    89  	major := pb & maskOutAdditionalType
    90  	minor := pb & maskOutMajorType
    91  	if major != majorTypeSimpleAndFloat {
    92  		panic(fmt.Errorf("Incorrect Major type is: %d in decodeFloat", major))
    93  	}
    94  
    95  	switch minor {
    96  	case additionalTypeFloat16:
    97  		panic(fmt.Errorf("float16 is not suppported in decodeFloat"))
    98  
    99  	case additionalTypeFloat32:
   100  		pb := readNBytes(src, 4)
   101  		switch string(pb) {
   102  		case float32Nan:
   103  			return math.NaN(), isFloat32
   104  		case float32PosInfinity:
   105  			return math.Inf(0), isFloat32
   106  		case float32NegInfinity:
   107  			return math.Inf(-1), isFloat32
   108  		}
   109  		n := uint32(0)
   110  		for i := 0; i < 4; i++ {
   111  			n = n * 256
   112  			n += uint32(pb[i])
   113  		}
   114  		val := math.Float32frombits(n)
   115  		return float64(val), isFloat32
   116  	case additionalTypeFloat64:
   117  		pb := readNBytes(src, 8)
   118  		switch string(pb) {
   119  		case float64Nan:
   120  			return math.NaN(), isFloat64
   121  		case float64PosInfinity:
   122  			return math.Inf(0), isFloat64
   123  		case float64NegInfinity:
   124  			return math.Inf(-1), isFloat64
   125  		}
   126  		n := uint64(0)
   127  		for i := 0; i < 8; i++ {
   128  			n = n * 256
   129  			n += uint64(pb[i])
   130  		}
   131  		val := math.Float64frombits(n)
   132  		return val, isFloat64
   133  	}
   134  	panic(fmt.Errorf("Invalid Additional Type: %d in decodeFloat", minor))
   135  }
   136  
   137  func decodeStringComplex(dst []byte, s string, pos uint) []byte {
   138  	i := int(pos)
   139  	start := 0
   140  
   141  	for i < len(s) {
   142  		b := s[i]
   143  		if b >= utf8.RuneSelf {
   144  			r, size := utf8.DecodeRuneInString(s[i:])
   145  			if r == utf8.RuneError && size == 1 {
   146  				// In case of error, first append previous simple characters to
   147  				// the byte slice if any and append a replacement character code
   148  				// in place of the invalid sequence.
   149  				if start < i {
   150  					dst = append(dst, s[start:i]...)
   151  				}
   152  				dst = append(dst, `\ufffd`...)
   153  				i += size
   154  				start = i
   155  				continue
   156  			}
   157  			i += size
   158  			continue
   159  		}
   160  		if b >= 0x20 && b <= 0x7e && b != '\\' && b != '"' {
   161  			i++
   162  			continue
   163  		}
   164  		// We encountered a character that needs to be encoded.
   165  		// Let's append the previous simple characters to the byte slice
   166  		// and switch our operation to read and encode the remainder
   167  		// characters byte-by-byte.
   168  		if start < i {
   169  			dst = append(dst, s[start:i]...)
   170  		}
   171  		switch b {
   172  		case '"', '\\':
   173  			dst = append(dst, '\\', b)
   174  		case '\b':
   175  			dst = append(dst, '\\', 'b')
   176  		case '\f':
   177  			dst = append(dst, '\\', 'f')
   178  		case '\n':
   179  			dst = append(dst, '\\', 'n')
   180  		case '\r':
   181  			dst = append(dst, '\\', 'r')
   182  		case '\t':
   183  			dst = append(dst, '\\', 't')
   184  		default:
   185  			dst = append(dst, '\\', 'u', '0', '0', hexTable[b>>4], hexTable[b&0xF])
   186  		}
   187  		i++
   188  		start = i
   189  	}
   190  	if start < len(s) {
   191  		dst = append(dst, s[start:]...)
   192  	}
   193  	return dst
   194  }
   195  
   196  func decodeString(src *bufio.Reader, noQuotes bool) []byte {
   197  	pb := readByte(src)
   198  	major := pb & maskOutAdditionalType
   199  	minor := pb & maskOutMajorType
   200  	if major != majorTypeByteString {
   201  		panic(fmt.Errorf("Major type is: %d in decodeString", major))
   202  	}
   203  	result := []byte{}
   204  	if !noQuotes {
   205  		result = append(result, '"')
   206  	}
   207  	length := decodeIntAdditionalType(src, minor)
   208  	len := int(length)
   209  	pbs := readNBytes(src, len)
   210  	result = append(result, pbs...)
   211  	if noQuotes {
   212  		return result
   213  	}
   214  	return append(result, '"')
   215  }
   216  
   217  func decodeUTF8String(src *bufio.Reader) []byte {
   218  	pb := readByte(src)
   219  	major := pb & maskOutAdditionalType
   220  	minor := pb & maskOutMajorType
   221  	if major != majorTypeUtf8String {
   222  		panic(fmt.Errorf("Major type is: %d in decodeUTF8String", major))
   223  	}
   224  	result := []byte{'"'}
   225  	length := decodeIntAdditionalType(src, minor)
   226  	len := int(length)
   227  	pbs := readNBytes(src, len)
   228  
   229  	for i := 0; i < len; i++ {
   230  		// Check if the character needs encoding. Control characters, slashes,
   231  		// and the double quote need json encoding. Bytes above the ascii
   232  		// boundary needs utf8 encoding.
   233  		if pbs[i] < 0x20 || pbs[i] > 0x7e || pbs[i] == '\\' || pbs[i] == '"' {
   234  			// We encountered a character that needs to be encoded. Switch
   235  			// to complex version of the algorithm.
   236  			dst := []byte{'"'}
   237  			dst = decodeStringComplex(dst, string(pbs), uint(i))
   238  			return append(dst, '"')
   239  		}
   240  	}
   241  	// The string has no need for encoding and therefore is directly
   242  	// appended to the byte slice.
   243  	result = append(result, pbs...)
   244  	return append(result, '"')
   245  }
   246  
   247  func array2Json(src *bufio.Reader, dst io.Writer) {
   248  	dst.Write([]byte{'['})
   249  	pb := readByte(src)
   250  	major := pb & maskOutAdditionalType
   251  	minor := pb & maskOutMajorType
   252  	if major != majorTypeArray {
   253  		panic(fmt.Errorf("Major type is: %d in array2Json", major))
   254  	}
   255  	len := 0
   256  	unSpecifiedCount := false
   257  	if minor == additionalTypeInfiniteCount {
   258  		unSpecifiedCount = true
   259  	} else {
   260  		length := decodeIntAdditionalType(src, minor)
   261  		len = int(length)
   262  	}
   263  	for i := 0; unSpecifiedCount || i < len; i++ {
   264  		if unSpecifiedCount {
   265  			pb, e := src.Peek(1)
   266  			if e != nil {
   267  				panic(e)
   268  			}
   269  			if pb[0] == majorTypeSimpleAndFloat|additionalTypeBreak {
   270  				readByte(src)
   271  				break
   272  			}
   273  		}
   274  		cbor2JsonOneObject(src, dst)
   275  		if unSpecifiedCount {
   276  			pb, e := src.Peek(1)
   277  			if e != nil {
   278  				panic(e)
   279  			}
   280  			if pb[0] == majorTypeSimpleAndFloat|additionalTypeBreak {
   281  				readByte(src)
   282  				break
   283  			}
   284  			dst.Write([]byte{','})
   285  		} else if i+1 < len {
   286  			dst.Write([]byte{','})
   287  		}
   288  	}
   289  	dst.Write([]byte{']'})
   290  }
   291  
   292  func map2Json(src *bufio.Reader, dst io.Writer) {
   293  	pb := readByte(src)
   294  	major := pb & maskOutAdditionalType
   295  	minor := pb & maskOutMajorType
   296  	if major != majorTypeMap {
   297  		panic(fmt.Errorf("Major type is: %d in map2Json", major))
   298  	}
   299  	len := 0
   300  	unSpecifiedCount := false
   301  	if minor == additionalTypeInfiniteCount {
   302  		unSpecifiedCount = true
   303  	} else {
   304  		length := decodeIntAdditionalType(src, minor)
   305  		len = int(length)
   306  	}
   307  	dst.Write([]byte{'{'})
   308  	for i := 0; unSpecifiedCount || i < len; i++ {
   309  		if unSpecifiedCount {
   310  			pb, e := src.Peek(1)
   311  			if e != nil {
   312  				panic(e)
   313  			}
   314  			if pb[0] == majorTypeSimpleAndFloat|additionalTypeBreak {
   315  				readByte(src)
   316  				break
   317  			}
   318  		}
   319  		cbor2JsonOneObject(src, dst)
   320  		if i%2 == 0 {
   321  			// Even position values are keys.
   322  			dst.Write([]byte{':'})
   323  		} else {
   324  			if unSpecifiedCount {
   325  				pb, e := src.Peek(1)
   326  				if e != nil {
   327  					panic(e)
   328  				}
   329  				if pb[0] == majorTypeSimpleAndFloat|additionalTypeBreak {
   330  					readByte(src)
   331  					break
   332  				}
   333  				dst.Write([]byte{','})
   334  			} else if i+1 < len {
   335  				dst.Write([]byte{','})
   336  			}
   337  		}
   338  	}
   339  	dst.Write([]byte{'}'})
   340  }
   341  
   342  func decodeTagData(src *bufio.Reader) []byte {
   343  	pb := readByte(src)
   344  	major := pb & maskOutAdditionalType
   345  	minor := pb & maskOutMajorType
   346  	if major != majorTypeTags {
   347  		panic(fmt.Errorf("Major type is: %d in decodeTagData", major))
   348  	}
   349  	switch minor {
   350  	case additionalTypeTimestamp:
   351  		return decodeTimeStamp(src)
   352  
   353  	// Tag value is larger than 256 (so uint16).
   354  	case additionalTypeIntUint16:
   355  		val := decodeIntAdditionalType(src, minor)
   356  
   357  		switch uint16(val) {
   358  		case additionalTypeEmbeddedJSON:
   359  			pb := readByte(src)
   360  			dataMajor := pb & maskOutAdditionalType
   361  			if dataMajor != majorTypeByteString {
   362  				panic(fmt.Errorf("Unsupported embedded Type: %d in decodeEmbeddedJSON", dataMajor))
   363  			}
   364  			src.UnreadByte()
   365  			return decodeString(src, true)
   366  
   367  		case additionalTypeTagNetworkAddr:
   368  			octets := decodeString(src, true)
   369  			ss := []byte{'"'}
   370  			switch len(octets) {
   371  			case 6: // MAC address.
   372  				ha := net.HardwareAddr(octets)
   373  				ss = append(append(ss, ha.String()...), '"')
   374  			case 4: // IPv4 address.
   375  				fallthrough
   376  			case 16: // IPv6 address.
   377  				ip := net.IP(octets)
   378  				ss = append(append(ss, ip.String()...), '"')
   379  			default:
   380  				panic(fmt.Errorf("Unexpected Network Address length: %d (expected 4,6,16)", len(octets)))
   381  			}
   382  			return ss
   383  
   384  		case additionalTypeTagNetworkPrefix:
   385  			pb := readByte(src)
   386  			if pb != majorTypeMap|0x1 {
   387  				panic(fmt.Errorf("IP Prefix is NOT of MAP of 1 elements as expected"))
   388  			}
   389  			octets := decodeString(src, true)
   390  			val := decodeInteger(src)
   391  			ip := net.IP(octets)
   392  			var mask net.IPMask
   393  			pfxLen := int(val)
   394  			if len(octets) == 4 {
   395  				mask = net.CIDRMask(pfxLen, 32)
   396  			} else {
   397  				mask = net.CIDRMask(pfxLen, 128)
   398  			}
   399  			ipPfx := net.IPNet{IP: ip, Mask: mask}
   400  			ss := []byte{'"'}
   401  			ss = append(append(ss, ipPfx.String()...), '"')
   402  			return ss
   403  
   404  		case additionalTypeTagHexString:
   405  			octets := decodeString(src, true)
   406  			ss := []byte{'"'}
   407  			for _, v := range octets {
   408  				ss = append(ss, hexTable[v>>4], hexTable[v&0x0f])
   409  			}
   410  			return append(ss, '"')
   411  
   412  		default:
   413  			panic(fmt.Errorf("Unsupported Additional Tag Type: %d in decodeTagData", val))
   414  		}
   415  	}
   416  	panic(fmt.Errorf("Unsupported Additional Type: %d in decodeTagData", minor))
   417  }
   418  
   419  func decodeTimeStamp(src *bufio.Reader) []byte {
   420  	pb := readByte(src)
   421  	src.UnreadByte()
   422  	tsMajor := pb & maskOutAdditionalType
   423  	if tsMajor == majorTypeUnsignedInt || tsMajor == majorTypeNegativeInt {
   424  		n := decodeInteger(src)
   425  		t := time.Unix(n, 0)
   426  		if decodeTimeZone != nil {
   427  			t = t.In(decodeTimeZone)
   428  		} else {
   429  			t = t.In(time.UTC)
   430  		}
   431  		tsb := []byte{}
   432  		tsb = append(tsb, '"')
   433  		tsb = t.AppendFormat(tsb, IntegerTimeFieldFormat)
   434  		tsb = append(tsb, '"')
   435  		return tsb
   436  	} else if tsMajor == majorTypeSimpleAndFloat {
   437  		n, _ := decodeFloat(src)
   438  		secs := int64(n)
   439  		n -= float64(secs)
   440  		n *= float64(1e9)
   441  		t := time.Unix(secs, int64(n))
   442  		if decodeTimeZone != nil {
   443  			t = t.In(decodeTimeZone)
   444  		} else {
   445  			t = t.In(time.UTC)
   446  		}
   447  		tsb := []byte{}
   448  		tsb = append(tsb, '"')
   449  		tsb = t.AppendFormat(tsb, NanoTimeFieldFormat)
   450  		tsb = append(tsb, '"')
   451  		return tsb
   452  	}
   453  	panic(fmt.Errorf("TS format is neigther int nor float: %d", tsMajor))
   454  }
   455  
   456  func decodeSimpleFloat(src *bufio.Reader) []byte {
   457  	pb := readByte(src)
   458  	major := pb & maskOutAdditionalType
   459  	minor := pb & maskOutMajorType
   460  	if major != majorTypeSimpleAndFloat {
   461  		panic(fmt.Errorf("Major type is: %d in decodeSimpleFloat", major))
   462  	}
   463  	switch minor {
   464  	case additionalTypeBoolTrue:
   465  		return []byte("true")
   466  	case additionalTypeBoolFalse:
   467  		return []byte("false")
   468  	case additionalTypeNull:
   469  		return []byte("null")
   470  	case additionalTypeFloat16:
   471  		fallthrough
   472  	case additionalTypeFloat32:
   473  		fallthrough
   474  	case additionalTypeFloat64:
   475  		src.UnreadByte()
   476  		v, bc := decodeFloat(src)
   477  		ba := []byte{}
   478  		switch {
   479  		case math.IsNaN(v):
   480  			return []byte("\"NaN\"")
   481  		case math.IsInf(v, 1):
   482  			return []byte("\"+Inf\"")
   483  		case math.IsInf(v, -1):
   484  			return []byte("\"-Inf\"")
   485  		}
   486  		if bc == isFloat32 {
   487  			ba = strconv.AppendFloat(ba, v, 'f', -1, 32)
   488  		} else if bc == isFloat64 {
   489  			ba = strconv.AppendFloat(ba, v, 'f', -1, 64)
   490  		} else {
   491  			panic(fmt.Errorf("Invalid Float precision from decodeFloat: %d", bc))
   492  		}
   493  		return ba
   494  	default:
   495  		panic(fmt.Errorf("Invalid Additional Type: %d in decodeSimpleFloat", minor))
   496  	}
   497  }
   498  
   499  func cbor2JsonOneObject(src *bufio.Reader, dst io.Writer) {
   500  	pb, e := src.Peek(1)
   501  	if e != nil {
   502  		panic(e)
   503  	}
   504  	major := (pb[0] & maskOutAdditionalType)
   505  
   506  	switch major {
   507  	case majorTypeUnsignedInt:
   508  		fallthrough
   509  	case majorTypeNegativeInt:
   510  		n := decodeInteger(src)
   511  		dst.Write([]byte(strconv.Itoa(int(n))))
   512  
   513  	case majorTypeByteString:
   514  		s := decodeString(src, false)
   515  		dst.Write(s)
   516  
   517  	case majorTypeUtf8String:
   518  		s := decodeUTF8String(src)
   519  		dst.Write(s)
   520  
   521  	case majorTypeArray:
   522  		array2Json(src, dst)
   523  
   524  	case majorTypeMap:
   525  		map2Json(src, dst)
   526  
   527  	case majorTypeTags:
   528  		s := decodeTagData(src)
   529  		dst.Write(s)
   530  
   531  	case majorTypeSimpleAndFloat:
   532  		s := decodeSimpleFloat(src)
   533  		dst.Write(s)
   534  	}
   535  }
   536  
   537  func moreBytesToRead(src *bufio.Reader) bool {
   538  	_, e := src.ReadByte()
   539  	if e == nil {
   540  		src.UnreadByte()
   541  		return true
   542  	}
   543  	return false
   544  }
   545  
   546  // Cbor2JsonManyObjects decodes all the CBOR Objects read from src
   547  // reader. It keeps on decoding until reader returns EOF (error when reading).
   548  // Decoded string is written to the dst. At the end of every CBOR Object
   549  // newline is written to the output stream.
   550  //
   551  // Returns error (if any) that was encountered during decode.
   552  // The child functions will generate a panic when error is encountered and
   553  // this function will recover non-runtime Errors and return the reason as error.
   554  func Cbor2JsonManyObjects(src io.Reader, dst io.Writer) (err error) {
   555  	defer func() {
   556  		if r := recover(); r != nil {
   557  			if _, ok := r.(runtime.Error); ok {
   558  				panic(r)
   559  			}
   560  			err = r.(error)
   561  		}
   562  	}()
   563  	bufRdr := bufio.NewReader(src)
   564  	for moreBytesToRead(bufRdr) {
   565  		cbor2JsonOneObject(bufRdr, dst)
   566  		dst.Write([]byte("\n"))
   567  	}
   568  	return nil
   569  }
   570  
   571  // Detect if the bytes to be printed is Binary or not.
   572  func binaryFmt(p []byte) bool {
   573  	if len(p) > 0 && p[0] > 0x7F {
   574  		return true
   575  	}
   576  	return false
   577  }
   578  
   579  func getReader(str string) *bufio.Reader {
   580  	return bufio.NewReader(strings.NewReader(str))
   581  }
   582  
   583  // DecodeIfBinaryToString converts a binary formatted log msg to a
   584  // JSON formatted String Log message - suitable for printing to Console/Syslog.
   585  func DecodeIfBinaryToString(in []byte) string {
   586  	if binaryFmt(in) {
   587  		var b bytes.Buffer
   588  		Cbor2JsonManyObjects(strings.NewReader(string(in)), &b)
   589  		return b.String()
   590  	}
   591  	return string(in)
   592  }
   593  
   594  // DecodeObjectToStr checks if the input is a binary format, if so,
   595  // it will decode a single Object and return the decoded string.
   596  func DecodeObjectToStr(in []byte) string {
   597  	if binaryFmt(in) {
   598  		var b bytes.Buffer
   599  		cbor2JsonOneObject(getReader(string(in)), &b)
   600  		return b.String()
   601  	}
   602  	return string(in)
   603  }
   604  
   605  // DecodeIfBinaryToBytes checks if the input is a binary format, if so,
   606  // it will decode all Objects and return the decoded string as byte array.
   607  func DecodeIfBinaryToBytes(in []byte) []byte {
   608  	if binaryFmt(in) {
   609  		var b bytes.Buffer
   610  		Cbor2JsonManyObjects(bytes.NewReader(in), &b)
   611  		return b.Bytes()
   612  	}
   613  	return in
   614  }
   615  

View as plain text