tar2ext4.go

Documentation: github.com/Microsoft/hcsshim/ext4/tar2ext4

     1  package tar2ext4
     2  
     3  import (
     4  	"archive/tar"
     5  	"bufio"
     6  	"encoding/binary"
     7  	"fmt"
     8  	"io"
     9  	"os"
    10  	"path"
    11  	"strings"
    12  
    13  	"github.com/Microsoft/hcsshim/ext4/dmverity"
    14  	"github.com/Microsoft/hcsshim/ext4/internal/compactext4"
    15  	"github.com/Microsoft/hcsshim/ext4/internal/format"
    16  	"github.com/pkg/errors"
    17  )
    18  
    19  type params struct {
    20  	convertWhiteout bool
    21  	appendVhdFooter bool
    22  	appendDMVerity  bool
    23  	ext4opts        []compactext4.Option
    24  }
    25  
    26  // Option is the type for optional parameters to Convert.
    27  type Option func(*params)
    28  
    29  // ConvertWhiteout instructs the converter to convert OCI-style whiteouts
    30  // (beginning with .wh.) to overlay-style whiteouts.
    31  func ConvertWhiteout(p *params) {
    32  	p.convertWhiteout = true
    33  }
    34  
    35  // AppendVhdFooter instructs the converter to add a fixed VHD footer to the
    36  // file.
    37  func AppendVhdFooter(p *params) {
    38  	p.appendVhdFooter = true
    39  }
    40  
    41  // AppendDMVerity instructs the converter to add a dmverity merkle tree for
    42  // the ext4 filesystem after the filesystem and before the optional VHD footer
    43  func AppendDMVerity(p *params) {
    44  	p.appendDMVerity = true
    45  }
    46  
    47  // InlineData instructs the converter to write small files into the inode
    48  // structures directly. This creates smaller images but currently is not
    49  // compatible with DAX.
    50  func InlineData(p *params) {
    51  	p.ext4opts = append(p.ext4opts, compactext4.InlineData)
    52  }
    53  
    54  // MaximumDiskSize instructs the writer to limit the disk size to the specified
    55  // value. This also reserves enough metadata space for the specified disk size.
    56  // If not provided, then 16GB is the default.
    57  func MaximumDiskSize(size int64) Option {
    58  	return func(p *params) {
    59  		p.ext4opts = append(p.ext4opts, compactext4.MaximumDiskSize(size))
    60  	}
    61  }
    62  
    63  const (
    64  	whiteoutPrefix = ".wh."
    65  	opaqueWhiteout = ".wh..wh..opq"
    66  )
    67  
    68  // ConvertTarToExt4 writes a compact ext4 file system image that contains the files in the
    69  // input tar stream.
    70  func ConvertTarToExt4(r io.Reader, w io.ReadWriteSeeker, options ...Option) error {
    71  	var p params
    72  	for _, opt := range options {
    73  		opt(&p)
    74  	}
    75  
    76  	t := tar.NewReader(bufio.NewReader(r))
    77  	fs := compactext4.NewWriter(w, p.ext4opts...)
    78  	for {
    79  		hdr, err := t.Next()
    80  		if err == io.EOF {
    81  			break
    82  		}
    83  		if err != nil {
    84  			return err
    85  		}
    86  
    87  		if err = fs.MakeParents(hdr.Name); err != nil {
    88  			return errors.Wrapf(err, "failed to ensure parent directories for %s", hdr.Name)
    89  		}
    90  
    91  		if p.convertWhiteout {
    92  			dir, name := path.Split(hdr.Name)
    93  			if strings.HasPrefix(name, whiteoutPrefix) {
    94  				if name == opaqueWhiteout {
    95  					// Update the directory with the appropriate xattr.
    96  					f, err := fs.Stat(dir)
    97  					if err != nil {
    98  						return errors.Wrapf(err, "failed to stat parent directory of whiteout %s", hdr.Name)
    99  					}
   100  					f.Xattrs["trusted.overlay.opaque"] = []byte("y")
   101  					err = fs.Create(dir, f)
   102  					if err != nil {
   103  						return errors.Wrapf(err, "failed to create opaque dir %s", hdr.Name)
   104  					}
   105  				} else {
   106  					// Create an overlay-style whiteout.
   107  					f := &compactext4.File{
   108  						Mode:     compactext4.S_IFCHR,
   109  						Devmajor: 0,
   110  						Devminor: 0,
   111  					}
   112  					err = fs.Create(path.Join(dir, name[len(whiteoutPrefix):]), f)
   113  					if err != nil {
   114  						return errors.Wrapf(err, "failed to create whiteout file for %s", hdr.Name)
   115  					}
   116  				}
   117  
   118  				continue
   119  			}
   120  		}
   121  
   122  		if hdr.Typeflag == tar.TypeLink {
   123  			err = fs.Link(hdr.Linkname, hdr.Name)
   124  			if err != nil {
   125  				return err
   126  			}
   127  		} else {
   128  			f := &compactext4.File{
   129  				Mode:     uint16(hdr.Mode),
   130  				Atime:    hdr.AccessTime,
   131  				Mtime:    hdr.ModTime,
   132  				Ctime:    hdr.ChangeTime,
   133  				Crtime:   hdr.ModTime,
   134  				Size:     hdr.Size,
   135  				Uid:      uint32(hdr.Uid),
   136  				Gid:      uint32(hdr.Gid),
   137  				Linkname: hdr.Linkname,
   138  				Devmajor: uint32(hdr.Devmajor),
   139  				Devminor: uint32(hdr.Devminor),
   140  				Xattrs:   make(map[string][]byte),
   141  			}
   142  			for key, value := range hdr.PAXRecords {
   143  				const xattrPrefix = "SCHILY.xattr."
   144  				if strings.HasPrefix(key, xattrPrefix) {
   145  					f.Xattrs[key[len(xattrPrefix):]] = []byte(value)
   146  				}
   147  			}
   148  
   149  			var typ uint16
   150  			switch hdr.Typeflag {
   151  			case tar.TypeReg, tar.TypeRegA:
   152  				typ = compactext4.S_IFREG
   153  			case tar.TypeSymlink:
   154  				typ = compactext4.S_IFLNK
   155  			case tar.TypeChar:
   156  				typ = compactext4.S_IFCHR
   157  			case tar.TypeBlock:
   158  				typ = compactext4.S_IFBLK
   159  			case tar.TypeDir:
   160  				typ = compactext4.S_IFDIR
   161  			case tar.TypeFifo:
   162  				typ = compactext4.S_IFIFO
   163  			}
   164  			f.Mode &= ^compactext4.TypeMask
   165  			f.Mode |= typ
   166  			err = fs.Create(hdr.Name, f)
   167  			if err != nil {
   168  				return err
   169  			}
   170  			_, err = io.Copy(fs, t)
   171  			if err != nil {
   172  				return err
   173  			}
   174  		}
   175  	}
   176  	return fs.Close()
   177  }
   178  
   179  // Convert wraps ConvertTarToExt4 and conditionally computes (and appends) the file image's cryptographic
   180  // hashes (merkle tree) or/and appends a VHD footer.
   181  func Convert(r io.Reader, w io.ReadWriteSeeker, options ...Option) error {
   182  	var p params
   183  	for _, opt := range options {
   184  		opt(&p)
   185  	}
   186  
   187  	if err := ConvertTarToExt4(r, w, options...); err != nil {
   188  		return err
   189  	}
   190  
   191  	if p.appendDMVerity {
   192  		if err := dmverity.ComputeAndWriteHashDevice(w, w); err != nil {
   193  			return err
   194  		}
   195  	}
   196  
   197  	if p.appendVhdFooter {
   198  		return ConvertToVhd(w)
   199  	}
   200  	return nil
   201  }
   202  
   203  // ReadExt4SuperBlock reads and returns ext4 super block from VHD
   204  //
   205  // The layout on disk is as follows:
   206  // | Group 0 padding     | - 1024 bytes
   207  // | ext4 SuperBlock     | - 1 block
   208  // | Group Descriptors   | - many blocks
   209  // | Reserved GDT Blocks | - many blocks
   210  // | Data Block Bitmap   | - 1 block
   211  // | inode Bitmap        | - 1 block
   212  // | inode Table         | - many blocks
   213  // | Data Blocks         | - many blocks
   214  //
   215  // More details can be found here https://ext4.wiki.kernel.org/index.php/Ext4_Disk_Layout
   216  //
   217  // Our goal is to skip the Group 0 padding, read and return the ext4 SuperBlock
   218  func ReadExt4SuperBlock(vhdPath string) (*format.SuperBlock, error) {
   219  	vhd, err := os.OpenFile(vhdPath, os.O_RDONLY, 0)
   220  	if err != nil {
   221  		return nil, err
   222  	}
   223  	defer vhd.Close()
   224  
   225  	// Skip padding at the start
   226  	if _, err := vhd.Seek(1024, io.SeekStart); err != nil {
   227  		return nil, err
   228  	}
   229  	var sb format.SuperBlock
   230  	if err := binary.Read(vhd, binary.LittleEndian, &sb); err != nil {
   231  		return nil, err
   232  	}
   233  	// Make sure the magic bytes are correct.
   234  	if sb.Magic != format.SuperBlockMagic {
   235  		return nil, errors.New("not an ext4 file system")
   236  	}
   237  	return &sb, nil
   238  }
   239  
   240  // ConvertAndComputeRootDigest writes a compact ext4 file system image that contains the files in the
   241  // input tar stream, computes the resulting file image's cryptographic hashes (merkle tree) and returns
   242  // merkle tree root digest. Convert is called with minimal options: ConvertWhiteout and MaximumDiskSize
   243  // set to dmverity.RecommendedVHDSizeGB.
   244  func ConvertAndComputeRootDigest(r io.Reader) (string, error) {
   245  	out, err := os.CreateTemp("", "")
   246  	if err != nil {
   247  		return "", fmt.Errorf("failed to create temporary file: %s", err)
   248  	}
   249  	defer func() {
   250  		_ = os.Remove(out.Name())
   251  	}()
   252  	defer out.Close()
   253  
   254  	options := []Option{
   255  		ConvertWhiteout,
   256  		MaximumDiskSize(dmverity.RecommendedVHDSizeGB),
   257  	}
   258  	if err := ConvertTarToExt4(r, out, options...); err != nil {
   259  		return "", fmt.Errorf("failed to convert tar to ext4: %s", err)
   260  	}
   261  
   262  	if _, err := out.Seek(0, io.SeekStart); err != nil {
   263  		return "", fmt.Errorf("failed to seek start on temp file when creating merkle tree: %s", err)
   264  	}
   265  
   266  	tree, err := dmverity.MerkleTree(bufio.NewReaderSize(out, dmverity.MerkleTreeBufioSize))
   267  	if err != nil {
   268  		return "", fmt.Errorf("failed to create merkle tree: %s", err)
   269  	}
   270  
   271  	hash := dmverity.RootHash(tree)
   272  	return fmt.Sprintf("%x", hash), nil
   273  }
   274  
   275  // ConvertToVhd converts given io.WriteSeeker to VHD, by appending the VHD footer with a fixed size.
   276  func ConvertToVhd(w io.WriteSeeker) error {
   277  	size, err := w.Seek(0, io.SeekEnd)
   278  	if err != nil {
   279  		return err
   280  	}
   281  	return binary.Write(w, binary.BigEndian, makeFixedVHDFooter(size))
   282  }
   283
View as plain text