Source file src/github.com/vbatts/tar-split/archive/tar/reader.go

Documentation: github.com/vbatts/tar-split/archive/tar

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package tar
     6  
     7  import (
     8  	"bytes"
     9  	"io"
    10  	"strconv"
    11  	"strings"
    12  	"time"
    13  )
    14  
    15  // Reader provides sequential access to the contents of a tar archive.
    16  // Reader.Next advances to the next file in the archive (including the first),
    17  // and then Reader can be treated as an io.Reader to access the file's data.
    18  type Reader struct {
    19  	r    io.Reader
    20  	pad  int64      // Amount of padding (ignored) after current file entry
    21  	curr fileReader // Reader for current file entry
    22  	blk  block      // Buffer to use as temporary local storage
    23  
    24  	// err is a persistent error.
    25  	// It is only the responsibility of every exported method of Reader to
    26  	// ensure that this error is sticky.
    27  	err error
    28  
    29  	RawAccounting bool          // Whether to enable the access needed to reassemble the tar from raw bytes. Some performance/memory hit for this.
    30  	rawBytes      *bytes.Buffer // last raw bits
    31  }
    32  
    33  type fileReader interface {
    34  	io.Reader
    35  	fileState
    36  
    37  	WriteTo(io.Writer) (int64, error)
    38  }
    39  
    40  // RawBytes accesses the raw bytes of the archive, apart from the file payload itself.
    41  // This includes the header and padding.
    42  //
    43  // # This call resets the current rawbytes buffer
    44  //
    45  // Only when RawAccounting is enabled, otherwise this returns nil
    46  func (tr *Reader) RawBytes() []byte {
    47  	if !tr.RawAccounting {
    48  		return nil
    49  	}
    50  	if tr.rawBytes == nil {
    51  		tr.rawBytes = bytes.NewBuffer(nil)
    52  	}
    53  	defer tr.rawBytes.Reset() // if we've read them, then flush them.
    54  
    55  	return tr.rawBytes.Bytes()
    56  
    57  }
    58  
    59  // NewReader creates a new Reader reading from r.
    60  func NewReader(r io.Reader) *Reader {
    61  	return &Reader{r: r, curr: &regFileReader{r, 0}}
    62  }
    63  
    64  // Next advances to the next entry in the tar archive.
    65  // The Header.Size determines how many bytes can be read for the next file.
    66  // Any remaining data in the current file is automatically discarded.
    67  //
    68  // io.EOF is returned at the end of the input.
    69  func (tr *Reader) Next() (*Header, error) {
    70  	if tr.err != nil {
    71  		return nil, tr.err
    72  	}
    73  	hdr, err := tr.next()
    74  	tr.err = err
    75  	return hdr, err
    76  }
    77  
    78  func (tr *Reader) next() (*Header, error) {
    79  	var paxHdrs map[string]string
    80  	var gnuLongName, gnuLongLink string
    81  
    82  	if tr.RawAccounting {
    83  		if tr.rawBytes == nil {
    84  			tr.rawBytes = bytes.NewBuffer(nil)
    85  		} else {
    86  			tr.rawBytes.Reset()
    87  		}
    88  	}
    89  
    90  	// Externally, Next iterates through the tar archive as if it is a series of
    91  	// files. Internally, the tar format often uses fake "files" to add meta
    92  	// data that describes the next file. These meta data "files" should not
    93  	// normally be visible to the outside. As such, this loop iterates through
    94  	// one or more "header files" until it finds a "normal file".
    95  	format := FormatUSTAR | FormatPAX | FormatGNU
    96  	for {
    97  		// Discard the remainder of the file and any padding.
    98  		if err := discard(tr, tr.curr.PhysicalRemaining()); err != nil {
    99  			return nil, err
   100  		}
   101  		n, err := tryReadFull(tr.r, tr.blk[:tr.pad])
   102  		if err != nil {
   103  			return nil, err
   104  		}
   105  		if tr.RawAccounting {
   106  			tr.rawBytes.Write(tr.blk[:n])
   107  		}
   108  		tr.pad = 0
   109  
   110  		hdr, rawHdr, err := tr.readHeader()
   111  		if err != nil {
   112  			return nil, err
   113  		}
   114  		if err := tr.handleRegularFile(hdr); err != nil {
   115  			return nil, err
   116  		}
   117  		format.mayOnlyBe(hdr.Format)
   118  
   119  		// Check for PAX/GNU special headers and files.
   120  		switch hdr.Typeflag {
   121  		case TypeXHeader, TypeXGlobalHeader:
   122  			format.mayOnlyBe(FormatPAX)
   123  			paxHdrs, err = parsePAX(tr)
   124  			if err != nil {
   125  				return nil, err
   126  			}
   127  			if hdr.Typeflag == TypeXGlobalHeader {
   128  				if err = mergePAX(hdr, paxHdrs); err != nil {
   129  					return nil, err
   130  				}
   131  				return &Header{
   132  					Name:       hdr.Name,
   133  					Typeflag:   hdr.Typeflag,
   134  					Xattrs:     hdr.Xattrs,
   135  					PAXRecords: hdr.PAXRecords,
   136  					Format:     format,
   137  				}, nil
   138  			}
   139  			continue // This is a meta header affecting the next header
   140  		case TypeGNULongName, TypeGNULongLink:
   141  			format.mayOnlyBe(FormatGNU)
   142  			realname, err := io.ReadAll(tr)
   143  			if err != nil {
   144  				return nil, err
   145  			}
   146  
   147  			if tr.RawAccounting {
   148  				tr.rawBytes.Write(realname)
   149  			}
   150  
   151  			var p parser
   152  			switch hdr.Typeflag {
   153  			case TypeGNULongName:
   154  				gnuLongName = p.parseString(realname)
   155  			case TypeGNULongLink:
   156  				gnuLongLink = p.parseString(realname)
   157  			}
   158  			continue // This is a meta header affecting the next header
   159  		default:
   160  			// The old GNU sparse format is handled here since it is technically
   161  			// just a regular file with additional attributes.
   162  
   163  			if err := mergePAX(hdr, paxHdrs); err != nil {
   164  				return nil, err
   165  			}
   166  			if gnuLongName != "" {
   167  				hdr.Name = gnuLongName
   168  			}
   169  			if gnuLongLink != "" {
   170  				hdr.Linkname = gnuLongLink
   171  			}
   172  			if hdr.Typeflag == TypeRegA {
   173  				if strings.HasSuffix(hdr.Name, "/") {
   174  					hdr.Typeflag = TypeDir // Legacy archives use trailing slash for directories
   175  				} else {
   176  					hdr.Typeflag = TypeReg
   177  				}
   178  			}
   179  
   180  			// The extended headers may have updated the size.
   181  			// Thus, setup the regFileReader again after merging PAX headers.
   182  			if err := tr.handleRegularFile(hdr); err != nil {
   183  				return nil, err
   184  			}
   185  
   186  			// Sparse formats rely on being able to read from the logical data
   187  			// section; there must be a preceding call to handleRegularFile.
   188  			if err := tr.handleSparseFile(hdr, rawHdr); err != nil {
   189  				return nil, err
   190  			}
   191  
   192  			// Set the final guess at the format.
   193  			if format.has(FormatUSTAR) && format.has(FormatPAX) {
   194  				format.mayOnlyBe(FormatUSTAR)
   195  			}
   196  			hdr.Format = format
   197  			return hdr, nil // This is a file, so stop
   198  		}
   199  	}
   200  }
   201  
   202  // handleRegularFile sets up the current file reader and padding such that it
   203  // can only read the following logical data section. It will properly handle
   204  // special headers that contain no data section.
   205  func (tr *Reader) handleRegularFile(hdr *Header) error {
   206  	nb := hdr.Size
   207  	if isHeaderOnlyType(hdr.Typeflag) {
   208  		nb = 0
   209  	}
   210  	if nb < 0 {
   211  		return ErrHeader
   212  	}
   213  
   214  	tr.pad = blockPadding(nb)
   215  	tr.curr = &regFileReader{r: tr.r, nb: nb}
   216  	return nil
   217  }
   218  
   219  // handleSparseFile checks if the current file is a sparse format of any type
   220  // and sets the curr reader appropriately.
   221  func (tr *Reader) handleSparseFile(hdr *Header, rawHdr *block) error {
   222  	var spd sparseDatas
   223  	var err error
   224  	if hdr.Typeflag == TypeGNUSparse {
   225  		spd, err = tr.readOldGNUSparseMap(hdr, rawHdr)
   226  	} else {
   227  		spd, err = tr.readGNUSparsePAXHeaders(hdr)
   228  	}
   229  
   230  	// If sp is non-nil, then this is a sparse file.
   231  	// Note that it is possible for len(sp) == 0.
   232  	if err == nil && spd != nil {
   233  		if isHeaderOnlyType(hdr.Typeflag) || !validateSparseEntries(spd, hdr.Size) {
   234  			return ErrHeader
   235  		}
   236  		sph := invertSparseEntries(spd, hdr.Size)
   237  		tr.curr = &sparseFileReader{tr.curr, sph, 0}
   238  	}
   239  	return err
   240  }
   241  
   242  // readGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers.
   243  // If they are found, then this function reads the sparse map and returns it.
   244  // This assumes that 0.0 headers have already been converted to 0.1 headers
   245  // by the PAX header parsing logic.
   246  func (tr *Reader) readGNUSparsePAXHeaders(hdr *Header) (sparseDatas, error) {
   247  	// Identify the version of GNU headers.
   248  	var is1x0 bool
   249  	major, minor := hdr.PAXRecords[paxGNUSparseMajor], hdr.PAXRecords[paxGNUSparseMinor]
   250  	switch {
   251  	case major == "0" && (minor == "0" || minor == "1"):
   252  		is1x0 = false
   253  	case major == "1" && minor == "0":
   254  		is1x0 = true
   255  	case major != "" || minor != "":
   256  		return nil, nil // Unknown GNU sparse PAX version
   257  	case hdr.PAXRecords[paxGNUSparseMap] != "":
   258  		is1x0 = false // 0.0 and 0.1 did not have explicit version records, so guess
   259  	default:
   260  		return nil, nil // Not a PAX format GNU sparse file.
   261  	}
   262  	hdr.Format.mayOnlyBe(FormatPAX)
   263  
   264  	// Update hdr from GNU sparse PAX headers.
   265  	if name := hdr.PAXRecords[paxGNUSparseName]; name != "" {
   266  		hdr.Name = name
   267  	}
   268  	size := hdr.PAXRecords[paxGNUSparseSize]
   269  	if size == "" {
   270  		size = hdr.PAXRecords[paxGNUSparseRealSize]
   271  	}
   272  	if size != "" {
   273  		n, err := strconv.ParseInt(size, 10, 64)
   274  		if err != nil {
   275  			return nil, ErrHeader
   276  		}
   277  		hdr.Size = n
   278  	}
   279  
   280  	// Read the sparse map according to the appropriate format.
   281  	if is1x0 {
   282  		return readGNUSparseMap1x0(tr.curr)
   283  	}
   284  	return readGNUSparseMap0x1(hdr.PAXRecords)
   285  }
   286  
   287  // mergePAX merges paxHdrs into hdr for all relevant fields of Header.
   288  func mergePAX(hdr *Header, paxHdrs map[string]string) (err error) {
   289  	for k, v := range paxHdrs {
   290  		if v == "" {
   291  			continue // Keep the original USTAR value
   292  		}
   293  		var id64 int64
   294  		switch k {
   295  		case paxPath:
   296  			hdr.Name = v
   297  		case paxLinkpath:
   298  			hdr.Linkname = v
   299  		case paxUname:
   300  			hdr.Uname = v
   301  		case paxGname:
   302  			hdr.Gname = v
   303  		case paxUid:
   304  			id64, err = strconv.ParseInt(v, 10, 64)
   305  			hdr.Uid = int(id64) // Integer overflow possible
   306  		case paxGid:
   307  			id64, err = strconv.ParseInt(v, 10, 64)
   308  			hdr.Gid = int(id64) // Integer overflow possible
   309  		case paxAtime:
   310  			hdr.AccessTime, err = parsePAXTime(v)
   311  		case paxMtime:
   312  			hdr.ModTime, err = parsePAXTime(v)
   313  		case paxCtime:
   314  			hdr.ChangeTime, err = parsePAXTime(v)
   315  		case paxSize:
   316  			hdr.Size, err = strconv.ParseInt(v, 10, 64)
   317  		default:
   318  			if strings.HasPrefix(k, paxSchilyXattr) {
   319  				if hdr.Xattrs == nil {
   320  					hdr.Xattrs = make(map[string]string)
   321  				}
   322  				hdr.Xattrs[k[len(paxSchilyXattr):]] = v
   323  			}
   324  		}
   325  		if err != nil {
   326  			return ErrHeader
   327  		}
   328  	}
   329  	hdr.PAXRecords = paxHdrs
   330  	return nil
   331  }
   332  
   333  // parsePAX parses PAX headers.
   334  // If an extended header (type 'x') is invalid, ErrHeader is returned
   335  func parsePAX(r io.Reader) (map[string]string, error) {
   336  	buf, err := io.ReadAll(r)
   337  	if err != nil {
   338  		return nil, err
   339  	}
   340  	// leaving this function for io.Reader makes it more testable
   341  	if tr, ok := r.(*Reader); ok && tr.RawAccounting {
   342  		if _, err = tr.rawBytes.Write(buf); err != nil {
   343  			return nil, err
   344  		}
   345  	}
   346  	sbuf := string(buf)
   347  
   348  	// For GNU PAX sparse format 0.0 support.
   349  	// This function transforms the sparse format 0.0 headers into format 0.1
   350  	// headers since 0.0 headers were not PAX compliant.
   351  	var sparseMap []string
   352  
   353  	paxHdrs := make(map[string]string)
   354  	for len(sbuf) > 0 {
   355  		key, value, residual, err := parsePAXRecord(sbuf)
   356  		if err != nil {
   357  			return nil, ErrHeader
   358  		}
   359  		sbuf = residual
   360  
   361  		switch key {
   362  		case paxGNUSparseOffset, paxGNUSparseNumBytes:
   363  			// Validate sparse header order and value.
   364  			if (len(sparseMap)%2 == 0 && key != paxGNUSparseOffset) ||
   365  				(len(sparseMap)%2 == 1 && key != paxGNUSparseNumBytes) ||
   366  				strings.Contains(value, ",") {
   367  				return nil, ErrHeader
   368  			}
   369  			sparseMap = append(sparseMap, value)
   370  		default:
   371  			paxHdrs[key] = value
   372  		}
   373  	}
   374  	if len(sparseMap) > 0 {
   375  		paxHdrs[paxGNUSparseMap] = strings.Join(sparseMap, ",")
   376  	}
   377  	return paxHdrs, nil
   378  }
   379  
   380  // readHeader reads the next block header and assumes that the underlying reader
   381  // is already aligned to a block boundary. It returns the raw block of the
   382  // header in case further processing is required.
   383  //
   384  // The err will be set to io.EOF only when one of the following occurs:
   385  //   - Exactly 0 bytes are read and EOF is hit.
   386  //   - Exactly 1 block of zeros is read and EOF is hit.
   387  //   - At least 2 blocks of zeros are read.
   388  func (tr *Reader) readHeader() (*Header, *block, error) {
   389  	// Two blocks of zero bytes marks the end of the archive.
   390  	n, err := io.ReadFull(tr.r, tr.blk[:])
   391  	if tr.RawAccounting && (err == nil || err == io.EOF) {
   392  		tr.rawBytes.Write(tr.blk[:n])
   393  	}
   394  	if err != nil {
   395  		return nil, nil, err // EOF is okay here; exactly 0 bytes read
   396  	}
   397  
   398  	if bytes.Equal(tr.blk[:], zeroBlock[:]) {
   399  		n, err = io.ReadFull(tr.r, tr.blk[:])
   400  		if tr.RawAccounting && (err == nil || err == io.EOF) {
   401  			tr.rawBytes.Write(tr.blk[:n])
   402  		}
   403  		if err != nil {
   404  			return nil, nil, err // EOF is okay here; exactly 1 block of zeros read
   405  		}
   406  		if bytes.Equal(tr.blk[:], zeroBlock[:]) {
   407  			return nil, nil, io.EOF // normal EOF; exactly 2 block of zeros read
   408  		}
   409  		return nil, nil, ErrHeader // Zero block and then non-zero block
   410  	}
   411  
   412  	// Verify the header matches a known format.
   413  	format := tr.blk.GetFormat()
   414  	if format == FormatUnknown {
   415  		return nil, nil, ErrHeader
   416  	}
   417  
   418  	var p parser
   419  	hdr := new(Header)
   420  
   421  	// Unpack the V7 header.
   422  	v7 := tr.blk.V7()
   423  	hdr.Typeflag = v7.TypeFlag()[0]
   424  	hdr.Name = p.parseString(v7.Name())
   425  	hdr.Linkname = p.parseString(v7.LinkName())
   426  	hdr.Size = p.parseNumeric(v7.Size())
   427  	hdr.Mode = p.parseNumeric(v7.Mode())
   428  	hdr.Uid = int(p.parseNumeric(v7.UID()))
   429  	hdr.Gid = int(p.parseNumeric(v7.GID()))
   430  	hdr.ModTime = time.Unix(p.parseNumeric(v7.ModTime()), 0)
   431  
   432  	// Unpack format specific fields.
   433  	if format > formatV7 {
   434  		ustar := tr.blk.USTAR()
   435  		hdr.Uname = p.parseString(ustar.UserName())
   436  		hdr.Gname = p.parseString(ustar.GroupName())
   437  		hdr.Devmajor = p.parseNumeric(ustar.DevMajor())
   438  		hdr.Devminor = p.parseNumeric(ustar.DevMinor())
   439  
   440  		var prefix string
   441  		switch {
   442  		case format.has(FormatUSTAR | FormatPAX):
   443  			hdr.Format = format
   444  			ustar := tr.blk.USTAR()
   445  			prefix = p.parseString(ustar.Prefix())
   446  
   447  			// For Format detection, check if block is properly formatted since
   448  			// the parser is more liberal than what USTAR actually permits.
   449  			notASCII := func(r rune) bool { return r >= 0x80 }
   450  			if bytes.IndexFunc(tr.blk[:], notASCII) >= 0 {
   451  				hdr.Format = FormatUnknown // Non-ASCII characters in block.
   452  			}
   453  			nul := func(b []byte) bool { return int(b[len(b)-1]) == 0 }
   454  			if !(nul(v7.Size()) && nul(v7.Mode()) && nul(v7.UID()) && nul(v7.GID()) &&
   455  				nul(v7.ModTime()) && nul(ustar.DevMajor()) && nul(ustar.DevMinor())) {
   456  				hdr.Format = FormatUnknown // Numeric fields must end in NUL
   457  			}
   458  		case format.has(formatSTAR):
   459  			star := tr.blk.STAR()
   460  			prefix = p.parseString(star.Prefix())
   461  			hdr.AccessTime = time.Unix(p.parseNumeric(star.AccessTime()), 0)
   462  			hdr.ChangeTime = time.Unix(p.parseNumeric(star.ChangeTime()), 0)
   463  		case format.has(FormatGNU):
   464  			hdr.Format = format
   465  			var p2 parser
   466  			gnu := tr.blk.GNU()
   467  			if b := gnu.AccessTime(); b[0] != 0 {
   468  				hdr.AccessTime = time.Unix(p2.parseNumeric(b), 0)
   469  			}
   470  			if b := gnu.ChangeTime(); b[0] != 0 {
   471  				hdr.ChangeTime = time.Unix(p2.parseNumeric(b), 0)
   472  			}
   473  
   474  			// Prior to Go1.8, the Writer had a bug where it would output
   475  			// an invalid tar file in certain rare situations because the logic
   476  			// incorrectly believed that the old GNU format had a prefix field.
   477  			// This is wrong and leads to an output file that mangles the
   478  			// atime and ctime fields, which are often left unused.
   479  			//
   480  			// In order to continue reading tar files created by former, buggy
   481  			// versions of Go, we skeptically parse the atime and ctime fields.
   482  			// If we are unable to parse them and the prefix field looks like
   483  			// an ASCII string, then we fallback on the pre-Go1.8 behavior
   484  			// of treating these fields as the USTAR prefix field.
   485  			//
   486  			// Note that this will not use the fallback logic for all possible
   487  			// files generated by a pre-Go1.8 toolchain. If the generated file
   488  			// happened to have a prefix field that parses as valid
   489  			// atime and ctime fields (e.g., when they are valid octal strings),
   490  			// then it is impossible to distinguish between an valid GNU file
   491  			// and an invalid pre-Go1.8 file.
   492  			//
   493  			// See https://golang.org/issues/12594
   494  			// See https://golang.org/issues/21005
   495  			if p2.err != nil {
   496  				hdr.AccessTime, hdr.ChangeTime = time.Time{}, time.Time{}
   497  				ustar := tr.blk.USTAR()
   498  				if s := p.parseString(ustar.Prefix()); isASCII(s) {
   499  					prefix = s
   500  				}
   501  				hdr.Format = FormatUnknown // Buggy file is not GNU
   502  			}
   503  		}
   504  		if len(prefix) > 0 {
   505  			hdr.Name = prefix + "/" + hdr.Name
   506  		}
   507  	}
   508  	return hdr, &tr.blk, p.err
   509  }
   510  
   511  // readOldGNUSparseMap reads the sparse map from the old GNU sparse format.
   512  // The sparse map is stored in the tar header if it's small enough.
   513  // If it's larger than four entries, then one or more extension headers are used
   514  // to store the rest of the sparse map.
   515  //
   516  // The Header.Size does not reflect the size of any extended headers used.
   517  // Thus, this function will read from the raw io.Reader to fetch extra headers.
   518  // This method mutates blk in the process.
   519  func (tr *Reader) readOldGNUSparseMap(hdr *Header, blk *block) (sparseDatas, error) {
   520  	// Make sure that the input format is GNU.
   521  	// Unfortunately, the STAR format also has a sparse header format that uses
   522  	// the same type flag but has a completely different layout.
   523  	if blk.GetFormat() != FormatGNU {
   524  		return nil, ErrHeader
   525  	}
   526  	hdr.Format.mayOnlyBe(FormatGNU)
   527  
   528  	var p parser
   529  	hdr.Size = p.parseNumeric(blk.GNU().RealSize())
   530  	if p.err != nil {
   531  		return nil, p.err
   532  	}
   533  	s := blk.GNU().Sparse()
   534  	spd := make(sparseDatas, 0, s.MaxEntries())
   535  	for {
   536  		for i := 0; i < s.MaxEntries(); i++ {
   537  			// This termination condition is identical to GNU and BSD tar.
   538  			if s.Entry(i).Offset()[0] == 0x00 {
   539  				break // Don't return, need to process extended headers (even if empty)
   540  			}
   541  			offset := p.parseNumeric(s.Entry(i).Offset())
   542  			length := p.parseNumeric(s.Entry(i).Length())
   543  			if p.err != nil {
   544  				return nil, p.err
   545  			}
   546  			spd = append(spd, sparseEntry{Offset: offset, Length: length})
   547  		}
   548  
   549  		if s.IsExtended()[0] > 0 {
   550  			// There are more entries. Read an extension header and parse its entries.
   551  			if _, err := mustReadFull(tr.r, blk[:]); err != nil {
   552  				return nil, err
   553  			}
   554  			if tr.RawAccounting {
   555  				tr.rawBytes.Write(blk[:])
   556  			}
   557  			s = blk.Sparse()
   558  			continue
   559  		}
   560  		return spd, nil // Done
   561  	}
   562  }
   563  
   564  // readGNUSparseMap1x0 reads the sparse map as stored in GNU's PAX sparse format
   565  // version 1.0. The format of the sparse map consists of a series of
   566  // newline-terminated numeric fields. The first field is the number of entries
   567  // and is always present. Following this are the entries, consisting of two
   568  // fields (offset, length). This function must stop reading at the end
   569  // boundary of the block containing the last newline.
   570  //
   571  // Note that the GNU manual says that numeric values should be encoded in octal
   572  // format. However, the GNU tar utility itself outputs these values in decimal.
   573  // As such, this library treats values as being encoded in decimal.
   574  func readGNUSparseMap1x0(r io.Reader) (sparseDatas, error) {
   575  	var (
   576  		cntNewline int64
   577  		buf        bytes.Buffer
   578  		blk        block
   579  	)
   580  
   581  	// feedTokens copies data in blocks from r into buf until there are
   582  	// at least cnt newlines in buf. It will not read more blocks than needed.
   583  	feedTokens := func(n int64) error {
   584  		for cntNewline < n {
   585  			if _, err := mustReadFull(r, blk[:]); err != nil {
   586  				return err
   587  			}
   588  			buf.Write(blk[:])
   589  			for _, c := range blk {
   590  				if c == '\n' {
   591  					cntNewline++
   592  				}
   593  			}
   594  		}
   595  		return nil
   596  	}
   597  
   598  	// nextToken gets the next token delimited by a newline. This assumes that
   599  	// at least one newline exists in the buffer.
   600  	nextToken := func() string {
   601  		cntNewline--
   602  		tok, _ := buf.ReadString('\n')
   603  		return strings.TrimRight(tok, "\n")
   604  	}
   605  
   606  	// Parse for the number of entries.
   607  	// Use integer overflow resistant math to check this.
   608  	if err := feedTokens(1); err != nil {
   609  		return nil, err
   610  	}
   611  	numEntries, err := strconv.ParseInt(nextToken(), 10, 0) // Intentionally parse as native int
   612  	if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) {
   613  		return nil, ErrHeader
   614  	}
   615  
   616  	// Parse for all member entries.
   617  	// numEntries is trusted after this since a potential attacker must have
   618  	// committed resources proportional to what this library used.
   619  	if err := feedTokens(2 * numEntries); err != nil {
   620  		return nil, err
   621  	}
   622  	spd := make(sparseDatas, 0, numEntries)
   623  	for i := int64(0); i < numEntries; i++ {
   624  		offset, err1 := strconv.ParseInt(nextToken(), 10, 64)
   625  		length, err2 := strconv.ParseInt(nextToken(), 10, 64)
   626  		if err1 != nil || err2 != nil {
   627  			return nil, ErrHeader
   628  		}
   629  		spd = append(spd, sparseEntry{Offset: offset, Length: length})
   630  	}
   631  	return spd, nil
   632  }
   633  
   634  // readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format
   635  // version 0.1. The sparse map is stored in the PAX headers.
   636  func readGNUSparseMap0x1(paxHdrs map[string]string) (sparseDatas, error) {
   637  	// Get number of entries.
   638  	// Use integer overflow resistant math to check this.
   639  	numEntriesStr := paxHdrs[paxGNUSparseNumBlocks]
   640  	numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0) // Intentionally parse as native int
   641  	if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) {
   642  		return nil, ErrHeader
   643  	}
   644  
   645  	// There should be two numbers in sparseMap for each entry.
   646  	sparseMap := strings.Split(paxHdrs[paxGNUSparseMap], ",")
   647  	if len(sparseMap) == 1 && sparseMap[0] == "" {
   648  		sparseMap = sparseMap[:0]
   649  	}
   650  	if int64(len(sparseMap)) != 2*numEntries {
   651  		return nil, ErrHeader
   652  	}
   653  
   654  	// Loop through the entries in the sparse map.
   655  	// numEntries is trusted now.
   656  	spd := make(sparseDatas, 0, numEntries)
   657  	for len(sparseMap) >= 2 {
   658  		offset, err1 := strconv.ParseInt(sparseMap[0], 10, 64)
   659  		length, err2 := strconv.ParseInt(sparseMap[1], 10, 64)
   660  		if err1 != nil || err2 != nil {
   661  			return nil, ErrHeader
   662  		}
   663  		spd = append(spd, sparseEntry{Offset: offset, Length: length})
   664  		sparseMap = sparseMap[2:]
   665  	}
   666  	return spd, nil
   667  }
   668  
   669  // Read reads from the current file in the tar archive.
   670  // It returns (0, io.EOF) when it reaches the end of that file,
   671  // until Next is called to advance to the next file.
   672  //
   673  // If the current file is sparse, then the regions marked as a hole
   674  // are read back as NUL-bytes.
   675  //
   676  // Calling Read on special types like TypeLink, TypeSymlink, TypeChar,
   677  // TypeBlock, TypeDir, and TypeFifo returns (0, io.EOF) regardless of what
   678  // the Header.Size claims.
   679  func (tr *Reader) Read(b []byte) (int, error) {
   680  	if tr.err != nil {
   681  		return 0, tr.err
   682  	}
   683  	n, err := tr.curr.Read(b)
   684  	if err != nil && err != io.EOF {
   685  		tr.err = err
   686  	}
   687  	return n, err
   688  }
   689  
   690  // writeTo writes the content of the current file to w.
   691  // The bytes written matches the number of remaining bytes in the current file.
   692  //
   693  // If the current file is sparse and w is an io.WriteSeeker,
   694  // then writeTo uses Seek to skip past holes defined in Header.SparseHoles,
   695  // assuming that skipped regions are filled with NULs.
   696  // This always writes the last byte to ensure w is the right size.
   697  //
   698  // TODO(dsnet): Re-export this when adding sparse file support.
   699  // See https://golang.org/issue/22735
   700  func (tr *Reader) writeTo(w io.Writer) (int64, error) {
   701  	if tr.err != nil {
   702  		return 0, tr.err
   703  	}
   704  	n, err := tr.curr.WriteTo(w)
   705  	if err != nil {
   706  		tr.err = err
   707  	}
   708  	return n, err
   709  }
   710  
   711  // regFileReader is a fileReader for reading data from a regular file entry.
   712  type regFileReader struct {
   713  	r  io.Reader // Underlying Reader
   714  	nb int64     // Number of remaining bytes to read
   715  }
   716  
   717  func (fr *regFileReader) Read(b []byte) (n int, err error) {
   718  	if int64(len(b)) > fr.nb {
   719  		b = b[:fr.nb]
   720  	}
   721  	if len(b) > 0 {
   722  		n, err = fr.r.Read(b)
   723  		fr.nb -= int64(n)
   724  	}
   725  	switch {
   726  	case err == io.EOF && fr.nb > 0:
   727  		return n, io.ErrUnexpectedEOF
   728  	case err == nil && fr.nb == 0:
   729  		return n, io.EOF
   730  	default:
   731  		return n, err
   732  	}
   733  }
   734  
   735  func (fr *regFileReader) WriteTo(w io.Writer) (int64, error) {
   736  	return io.Copy(w, struct{ io.Reader }{fr})
   737  }
   738  
   739  func (fr regFileReader) LogicalRemaining() int64 {
   740  	return fr.nb
   741  }
   742  
   743  func (fr regFileReader) PhysicalRemaining() int64 {
   744  	return fr.nb
   745  }
   746  
   747  // sparseFileReader is a fileReader for reading data from a sparse file entry.
   748  type sparseFileReader struct {
   749  	fr  fileReader  // Underlying fileReader
   750  	sp  sparseHoles // Normalized list of sparse holes
   751  	pos int64       // Current position in sparse file
   752  }
   753  
   754  func (sr *sparseFileReader) Read(b []byte) (n int, err error) {
   755  	finished := int64(len(b)) >= sr.LogicalRemaining()
   756  	if finished {
   757  		b = b[:sr.LogicalRemaining()]
   758  	}
   759  
   760  	b0 := b
   761  	endPos := sr.pos + int64(len(b))
   762  	for endPos > sr.pos && err == nil {
   763  		var nf int // Bytes read in fragment
   764  		holeStart, holeEnd := sr.sp[0].Offset, sr.sp[0].endOffset()
   765  		if sr.pos < holeStart { // In a data fragment
   766  			bf := b[:min(int64(len(b)), holeStart-sr.pos)]
   767  			nf, err = tryReadFull(sr.fr, bf)
   768  		} else { // In a hole fragment
   769  			bf := b[:min(int64(len(b)), holeEnd-sr.pos)]
   770  			nf, err = tryReadFull(zeroReader{}, bf)
   771  		}
   772  		b = b[nf:]
   773  		sr.pos += int64(nf)
   774  		if sr.pos >= holeEnd && len(sr.sp) > 1 {
   775  			sr.sp = sr.sp[1:] // Ensure last fragment always remains
   776  		}
   777  	}
   778  
   779  	n = len(b0) - len(b)
   780  	switch {
   781  	case err == io.EOF:
   782  		return n, errMissData // Less data in dense file than sparse file
   783  	case err != nil:
   784  		return n, err
   785  	case sr.LogicalRemaining() == 0 && sr.PhysicalRemaining() > 0:
   786  		return n, errUnrefData // More data in dense file than sparse file
   787  	case finished:
   788  		return n, io.EOF
   789  	default:
   790  		return n, nil
   791  	}
   792  }
   793  
   794  func (sr *sparseFileReader) WriteTo(w io.Writer) (n int64, err error) {
   795  	ws, ok := w.(io.WriteSeeker)
   796  	if ok {
   797  		if _, err := ws.Seek(0, io.SeekCurrent); err != nil {
   798  			ok = false // Not all io.Seeker can really seek
   799  		}
   800  	}
   801  	if !ok {
   802  		return io.Copy(w, struct{ io.Reader }{sr})
   803  	}
   804  
   805  	var writeLastByte bool
   806  	pos0 := sr.pos
   807  	for sr.LogicalRemaining() > 0 && !writeLastByte && err == nil {
   808  		var nf int64 // Size of fragment
   809  		holeStart, holeEnd := sr.sp[0].Offset, sr.sp[0].endOffset()
   810  		if sr.pos < holeStart { // In a data fragment
   811  			nf = holeStart - sr.pos
   812  			nf, err = io.CopyN(ws, sr.fr, nf)
   813  		} else { // In a hole fragment
   814  			nf = holeEnd - sr.pos
   815  			if sr.PhysicalRemaining() == 0 {
   816  				writeLastByte = true
   817  				nf--
   818  			}
   819  			_, err = ws.Seek(nf, io.SeekCurrent)
   820  		}
   821  		sr.pos += nf
   822  		if sr.pos >= holeEnd && len(sr.sp) > 1 {
   823  			sr.sp = sr.sp[1:] // Ensure last fragment always remains
   824  		}
   825  	}
   826  
   827  	// If the last fragment is a hole, then seek to 1-byte before EOF, and
   828  	// write a single byte to ensure the file is the right size.
   829  	if writeLastByte && err == nil {
   830  		_, err = ws.Write([]byte{0})
   831  		sr.pos++
   832  	}
   833  
   834  	n = sr.pos - pos0
   835  	switch {
   836  	case err == io.EOF:
   837  		return n, errMissData // Less data in dense file than sparse file
   838  	case err != nil:
   839  		return n, err
   840  	case sr.LogicalRemaining() == 0 && sr.PhysicalRemaining() > 0:
   841  		return n, errUnrefData // More data in dense file than sparse file
   842  	default:
   843  		return n, nil
   844  	}
   845  }
   846  
   847  func (sr sparseFileReader) LogicalRemaining() int64 {
   848  	return sr.sp[len(sr.sp)-1].endOffset() - sr.pos
   849  }
   850  func (sr sparseFileReader) PhysicalRemaining() int64 {
   851  	return sr.fr.PhysicalRemaining()
   852  }
   853  
   854  type zeroReader struct{}
   855  
   856  func (zeroReader) Read(b []byte) (int, error) {
   857  	for i := range b {
   858  		b[i] = 0
   859  	}
   860  	return len(b), nil
   861  }
   862  
   863  // mustReadFull is like io.ReadFull except it returns
   864  // io.ErrUnexpectedEOF when io.EOF is hit before len(b) bytes are read.
   865  func mustReadFull(r io.Reader, b []byte) (int, error) {
   866  	n, err := tryReadFull(r, b)
   867  	if err == io.EOF {
   868  		err = io.ErrUnexpectedEOF
   869  	}
   870  	return n, err
   871  }
   872  
   873  // tryReadFull is like io.ReadFull except it returns
   874  // io.EOF when it is hit before len(b) bytes are read.
   875  func tryReadFull(r io.Reader, b []byte) (n int, err error) {
   876  	for len(b) > n && err == nil {
   877  		var nn int
   878  		nn, err = r.Read(b[n:])
   879  		n += nn
   880  	}
   881  	if len(b) == n && err == io.EOF {
   882  		err = nil
   883  	}
   884  	return n, err
   885  }
   886  
   887  // discard skips n bytes in r, reporting an error if unable to do so.
   888  func discard(tr *Reader, n int64) error {
   889  	var seekSkipped, copySkipped int64
   890  	var err error
   891  	r := tr.r
   892  	if tr.RawAccounting {
   893  
   894  		copySkipped, err = io.CopyN(tr.rawBytes, tr.r, n)
   895  		goto out
   896  	}
   897  
   898  	// If possible, Seek to the last byte before the end of the data section.
   899  	// Do this because Seek is often lazy about reporting errors; this will mask
   900  	// the fact that the stream may be truncated. We can rely on the
   901  	// io.CopyN done shortly afterwards to trigger any IO errors.
   902  	if sr, ok := r.(io.Seeker); ok && n > 1 {
   903  		// Not all io.Seeker can actually Seek. For example, os.Stdin implements
   904  		// io.Seeker, but calling Seek always returns an error and performs
   905  		// no action. Thus, we try an innocent seek to the current position
   906  		// to see if Seek is really supported.
   907  		pos1, err := sr.Seek(0, io.SeekCurrent)
   908  		if pos1 >= 0 && err == nil {
   909  			// Seek seems supported, so perform the real Seek.
   910  			pos2, err := sr.Seek(n-1, io.SeekCurrent)
   911  			if pos2 < 0 || err != nil {
   912  				return err
   913  			}
   914  			seekSkipped = pos2 - pos1
   915  		}
   916  	}
   917  
   918  	copySkipped, err = io.CopyN(io.Discard, r, n-seekSkipped)
   919  out:
   920  	if err == io.EOF && seekSkipped+copySkipped < n {
   921  		err = io.ErrUnexpectedEOF
   922  	}
   923  	return err
   924  }
   925  

View as plain text