assemble_test.go

Documentation: github.com/vbatts/tar-split/tar/asm

     1  package asm
     2  
     3  import (
     4  	"bytes"
     5  	"compress/gzip"
     6  	"crypto/sha1"
     7  	"fmt"
     8  	"hash/crc64"
     9  	"io"
    10  	"os"
    11  	"testing"
    12  
    13  	"github.com/vbatts/tar-split/tar/storage"
    14  )
    15  
    16  var entries = []struct {
    17  	Entry storage.Entry
    18  	Body  []byte
    19  }{
    20  	{
    21  		Entry: storage.Entry{
    22  			Type:    storage.FileType,
    23  			Name:    "./hurr.txt",
    24  			Payload: []byte{2, 116, 164, 177, 171, 236, 107, 78},
    25  			Size:    20,
    26  		},
    27  		Body: []byte("imma hurr til I derp"),
    28  	},
    29  	{
    30  		Entry: storage.Entry{
    31  			Type:    storage.FileType,
    32  			Name:    "./ermahgerd.txt",
    33  			Payload: []byte{126, 72, 89, 239, 230, 252, 160, 187},
    34  			Size:    26,
    35  		},
    36  		Body: []byte("café con leche, por favor"),
    37  	},
    38  	{
    39  		Entry: storage.Entry{
    40  			Type:    storage.FileType,
    41  			NameRaw: []byte{0x66, 0x69, 0x6c, 0x65, 0x2d, 0xe4}, // this is invalid UTF-8. Just checking the round trip.
    42  			Payload: []byte{126, 72, 89, 239, 230, 252, 160, 187},
    43  			Size:    26,
    44  		},
    45  		Body: []byte("café con leche, por favor"),
    46  	},
    47  }
    48  var entriesMangled = []struct {
    49  	Entry storage.Entry
    50  	Body  []byte
    51  }{
    52  	{
    53  		Entry: storage.Entry{
    54  			Type:    storage.FileType,
    55  			Name:    "./hurr.txt",
    56  			Payload: []byte{3, 116, 164, 177, 171, 236, 107, 78},
    57  			Size:    20,
    58  		},
    59  		// switch
    60  		Body: []byte("imma derp til I hurr"),
    61  	},
    62  	{
    63  		Entry: storage.Entry{
    64  			Type:    storage.FileType,
    65  			Name:    "./ermahgerd.txt",
    66  			Payload: []byte{127, 72, 89, 239, 230, 252, 160, 187},
    67  			Size:    26,
    68  		},
    69  		// san not con
    70  		Body: []byte("café sans leche, por favor"),
    71  	},
    72  	{
    73  		Entry: storage.Entry{
    74  			Type:    storage.FileType,
    75  			NameRaw: []byte{0x66, 0x69, 0x6c, 0x65, 0x2d, 0xe4},
    76  			Payload: []byte{127, 72, 89, 239, 230, 252, 160, 187},
    77  			Size:    26,
    78  		},
    79  		Body: []byte("café con leche, por favor"),
    80  	},
    81  }
    82  
    83  func TestTarStreamMangledGetterPutter(t *testing.T) {
    84  	fgp := storage.NewBufferFileGetPutter()
    85  
    86  	// first lets prep a GetPutter and Packer
    87  	for i := range entries {
    88  		if entries[i].Entry.Type == storage.FileType {
    89  			j, csum, err := fgp.Put(entries[i].Entry.GetName(), bytes.NewBuffer(entries[i].Body))
    90  			if err != nil {
    91  				t.Error(err)
    92  			}
    93  			if j != entries[i].Entry.Size {
    94  				t.Errorf("size %q: expected %d; got %d",
    95  					entries[i].Entry.GetName(),
    96  					entries[i].Entry.Size,
    97  					j)
    98  			}
    99  			if !bytes.Equal(csum, entries[i].Entry.Payload) {
   100  				t.Errorf("checksum %q: expected %v; got %v",
   101  					entries[i].Entry.GetName(),
   102  					entries[i].Entry.Payload,
   103  					csum)
   104  			}
   105  		}
   106  	}
   107  
   108  	for _, e := range entriesMangled {
   109  		if e.Entry.Type == storage.FileType {
   110  			rdr, err := fgp.Get(e.Entry.GetName())
   111  			if err != nil {
   112  				t.Error(err)
   113  			}
   114  			c := crc64.New(storage.CRCTable)
   115  			i, err := io.Copy(c, rdr)
   116  			if err != nil {
   117  				t.Fatal(err)
   118  			}
   119  			rdr.Close()
   120  
   121  			csum := c.Sum(nil)
   122  			if bytes.Equal(csum, e.Entry.Payload) {
   123  				t.Errorf("wrote %d bytes. checksum for %q should not have matched! %v",
   124  					i,
   125  					e.Entry.GetName(),
   126  					csum)
   127  			}
   128  		}
   129  	}
   130  }
   131  
   132  var testCases = []struct {
   133  	path            string
   134  	expectedSHA1Sum string
   135  	expectedSize    int64
   136  }{
   137  	{"./testdata/t.tar.gz", "1eb237ff69bca6e22789ecb05b45d35ca307adbd", 10240},
   138  	{"./testdata/longlink.tar.gz", "d9f6babe107b7247953dff6b5b5ae31a3a880add", 20480},
   139  	{"./testdata/fatlonglink.tar.gz", "8537f03f89aeef537382f8b0bb065d93e03b0be8", 26234880},
   140  	{"./testdata/iso-8859.tar.gz", "ddafa51cb03c74ec117ab366ee2240d13bba1ec3", 10240},
   141  	{"./testdata/extranils.tar.gz", "e187b4b3e739deaccc257342f4940f34403dc588", 10648},
   142  	{"./testdata/notenoughnils.tar.gz", "72f93f41efd95290baa5c174c234f5d4c22ce601", 512},
   143  	{"./testdata/1c51fc286aa95d9413226599576bafa38490b1e292375c90de095855b64caea6", "946caa03167a8cc707db6ff9785608b652e631dc", 1024},
   144  }
   145  
   146  func TestTarStream(t *testing.T) {
   147  
   148  	for _, tc := range testCases {
   149  		fh, err := os.Open(tc.path)
   150  		if err != nil {
   151  			t.Fatal(err)
   152  		}
   153  		defer fh.Close()
   154  		gzRdr, err := gzip.NewReader(fh)
   155  		if err != nil {
   156  			t.Fatal(err)
   157  		}
   158  		defer gzRdr.Close()
   159  
   160  		// Setup where we'll store the metadata
   161  		w := bytes.NewBuffer([]byte{})
   162  		sp := storage.NewJSONPacker(w)
   163  		fgp := storage.NewBufferFileGetPutter()
   164  
   165  		// wrap the disassembly stream
   166  		tarStream, err := NewInputTarStream(gzRdr, sp, fgp)
   167  		if err != nil {
   168  			t.Fatal(err)
   169  		}
   170  
   171  		// get a sum of the stream after it has passed through to ensure it's the same.
   172  		h0 := sha1.New()
   173  		i, err := io.Copy(h0, tarStream)
   174  		if err != nil {
   175  			t.Fatal(err)
   176  		}
   177  
   178  		if i != tc.expectedSize {
   179  			t.Errorf("size of tar: expected %d; got %d", tc.expectedSize, i)
   180  		}
   181  		if fmt.Sprintf("%x", h0.Sum(nil)) != tc.expectedSHA1Sum {
   182  			t.Fatalf("checksum of tar: expected %s; got %x", tc.expectedSHA1Sum, h0.Sum(nil))
   183  		}
   184  
   185  		//t.Logf("%s", w.String()) // if we fail, then show the packed info
   186  
   187  		// If we've made it this far, then we'll turn it around and create a tar
   188  		// stream from the packed metadata and buffered file contents.
   189  		r := bytes.NewBuffer(w.Bytes())
   190  		sup := storage.NewJSONUnpacker(r)
   191  		// and reuse the fgp that we Put the payloads to.
   192  
   193  		rc := NewOutputTarStream(fgp, sup)
   194  		h1 := sha1.New()
   195  		i, err = io.Copy(h1, rc)
   196  		if err != nil {
   197  			t.Fatal(err)
   198  		}
   199  
   200  		if i != tc.expectedSize {
   201  			t.Errorf("size of output tar: expected %d; got %d", tc.expectedSize, i)
   202  		}
   203  		if fmt.Sprintf("%x", h1.Sum(nil)) != tc.expectedSHA1Sum {
   204  			t.Fatalf("checksum of output tar: expected %s; got %x", tc.expectedSHA1Sum, h1.Sum(nil))
   205  		}
   206  	}
   207  }
   208  
   209  func BenchmarkAsm(b *testing.B) {
   210  	for i := 0; i < b.N; i++ {
   211  		for _, tc := range testCases {
   212  			func() {
   213  				fh, err := os.Open(tc.path)
   214  				if err != nil {
   215  					b.Fatal(err)
   216  				}
   217  				defer fh.Close()
   218  				gzRdr, err := gzip.NewReader(fh)
   219  				if err != nil {
   220  					b.Fatal(err)
   221  				}
   222  				defer gzRdr.Close()
   223  
   224  				// Setup where we'll store the metadata
   225  				w := bytes.NewBuffer([]byte{})
   226  				sp := storage.NewJSONPacker(w)
   227  				fgp := storage.NewBufferFileGetPutter()
   228  
   229  				// wrap the disassembly stream
   230  				tarStream, err := NewInputTarStream(gzRdr, sp, fgp)
   231  				if err != nil {
   232  					b.Fatal(err)
   233  				}
   234  				// read it all to the bit bucket
   235  				i1, err := io.Copy(io.Discard, tarStream)
   236  				if err != nil {
   237  					b.Fatal(err)
   238  				}
   239  
   240  				r := bytes.NewBuffer(w.Bytes())
   241  				sup := storage.NewJSONUnpacker(r)
   242  				// and reuse the fgp that we Put the payloads to.
   243  
   244  				rc := NewOutputTarStream(fgp, sup)
   245  
   246  				i2, err := io.Copy(io.Discard, rc)
   247  				if err != nil {
   248  					b.Fatal(err)
   249  				}
   250  				if i1 != i2 {
   251  					b.Errorf("%s: input(%d) and ouput(%d) byte count didn't match", tc.path, i1, i2)
   252  				}
   253  			}()
   254  		}
   255  	}
   256  }
   257
View as plain text