progress_test.go

Documentation: github.com/google/go-containerregistry/pkg/v1/remote

     1  // Copyright 2021 Google LLC All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //    http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package remote
    16  
    17  import (
    18  	"fmt"
    19  	"net/http"
    20  	"net/http/httptest"
    21  	"net/url"
    22  	"strings"
    23  	"sync"
    24  	"testing"
    25  
    26  	"github.com/google/go-cmp/cmp"
    27  	"github.com/google/go-containerregistry/pkg/name"
    28  	"github.com/google/go-containerregistry/pkg/registry"
    29  	v1 "github.com/google/go-containerregistry/pkg/v1"
    30  	"github.com/google/go-containerregistry/pkg/v1/empty"
    31  	"github.com/google/go-containerregistry/pkg/v1/mutate"
    32  	"github.com/google/go-containerregistry/pkg/v1/random"
    33  	"github.com/google/go-containerregistry/pkg/v1/types"
    34  )
    35  
    36  func TestWriteLayer_Progress(t *testing.T) {
    37  	l, err := random.Layer(1000, types.OCIUncompressedLayer)
    38  	if err != nil {
    39  		t.Fatal(err)
    40  	}
    41  	c := make(chan v1.Update, 200)
    42  
    43  	// Set up a fake registry.
    44  	s := httptest.NewServer(registry.New())
    45  	defer s.Close()
    46  	u, err := url.Parse(s.URL)
    47  	if err != nil {
    48  		t.Fatal(err)
    49  	}
    50  	dst := fmt.Sprintf("%s/test/progress/upload", u.Host)
    51  	ref, err := name.ParseReference(dst)
    52  	if err != nil {
    53  		t.Fatal(err)
    54  	}
    55  
    56  	if err := WriteLayer(ref.Context(), l, WithProgress(c)); err != nil {
    57  		t.Fatalf("WriteLayer: %v", err)
    58  	}
    59  	if err := checkUpdates(c); err != nil {
    60  		t.Fatal(err)
    61  	}
    62  }
    63  
    64  // TestWriteLayer_Progress_Exists tests progress reporting behavior when the
    65  // layer already exists in the registry, so writes are skipped, but progress
    66  // should still be reported in one update.
    67  func TestWriteLayer_Progress_Exists(t *testing.T) {
    68  	l, err := random.Layer(1000, types.OCILayer)
    69  	if err != nil {
    70  		t.Fatal(err)
    71  	}
    72  	c := make(chan v1.Update, 200)
    73  
    74  	// Set up a fake registry.
    75  	s := httptest.NewServer(registry.New())
    76  	defer s.Close()
    77  	u, err := url.Parse(s.URL)
    78  	if err != nil {
    79  		t.Fatal(err)
    80  	}
    81  	dst := fmt.Sprintf("%s/test/progress/upload", u.Host)
    82  	ref, err := name.ParseReference(dst)
    83  	if err != nil {
    84  		t.Fatal(err)
    85  	}
    86  
    87  	// Write the layer, so we can get updates when we write it again.
    88  	if err := WriteLayer(ref.Context(), l); err != nil {
    89  		t.Fatalf("WriteLayer: %v", err)
    90  	}
    91  	if err := WriteLayer(ref.Context(), l, WithProgress(c)); err != nil {
    92  		t.Fatalf("WriteLayer: %v", err)
    93  	}
    94  	if err := checkUpdates(c); err != nil {
    95  		t.Fatal(err)
    96  	}
    97  }
    98  
    99  func TestWrite_Progress(t *testing.T) {
   100  	img, err := random.Image(1000, 5)
   101  	if err != nil {
   102  		t.Fatal(err)
   103  	}
   104  	c := make(chan v1.Update, 200)
   105  
   106  	// Set up a fake registry.
   107  	s := httptest.NewServer(registry.New())
   108  	defer s.Close()
   109  	u, err := url.Parse(s.URL)
   110  	if err != nil {
   111  		t.Fatal(err)
   112  	}
   113  	dst := fmt.Sprintf("%s/test/progress/upload", u.Host)
   114  	ref, err := name.ParseReference(dst)
   115  	if err != nil {
   116  		t.Fatal(err)
   117  	}
   118  
   119  	if err := Write(ref, img, WithProgress(c)); err != nil {
   120  		t.Fatalf("Write: %v", err)
   121  	}
   122  
   123  	if err := checkUpdates(c); err != nil {
   124  		t.Fatal(err)
   125  	}
   126  }
   127  
   128  // An image with multiple identical layers is handled correctly.
   129  func TestWrite_Progress_DedupeLayers(t *testing.T) {
   130  	img := empty.Image
   131  	for i := 0; i < 10; i++ {
   132  		l, err := random.Layer(1000, types.OCILayer)
   133  		if err != nil {
   134  			t.Fatal(err)
   135  		}
   136  
   137  		img, err = mutate.AppendLayers(img, l)
   138  		if err != nil {
   139  			t.Fatal(err)
   140  		}
   141  	}
   142  
   143  	c := make(chan v1.Update, 200)
   144  
   145  	// Set up a fake registry.
   146  	s := httptest.NewServer(registry.New())
   147  	defer s.Close()
   148  	u, err := url.Parse(s.URL)
   149  	if err != nil {
   150  		t.Fatal(err)
   151  	}
   152  	dst := fmt.Sprintf("%s/test/progress/upload", u.Host)
   153  	ref, err := name.ParseReference(dst)
   154  	if err != nil {
   155  		t.Fatal(err)
   156  	}
   157  
   158  	if err := Write(ref, img, WithProgress(c)); err != nil {
   159  		t.Fatalf("Write: %v", err)
   160  	}
   161  
   162  	if err := checkUpdates(c); err != nil {
   163  		t.Fatal(err)
   164  	}
   165  }
   166  
   167  func TestWriteIndex_Progress(t *testing.T) {
   168  	idx, err := random.Index(1000, 3, 3)
   169  	if err != nil {
   170  		t.Fatal(err)
   171  	}
   172  	c := make(chan v1.Update, 200)
   173  
   174  	// Set up a fake registry.
   175  	s := httptest.NewServer(registry.New())
   176  	defer s.Close()
   177  	u, err := url.Parse(s.URL)
   178  	if err != nil {
   179  		t.Fatal(err)
   180  	}
   181  	dst := fmt.Sprintf("%s/test/progress/upload", u.Host)
   182  	ref, err := name.ParseReference(dst)
   183  	if err != nil {
   184  		t.Fatal(err)
   185  	}
   186  
   187  	if err := WriteIndex(ref, idx, WithProgress(c)); err != nil {
   188  		t.Fatalf("WriteIndex: %v", err)
   189  	}
   190  
   191  	if err := checkUpdates(c); err != nil {
   192  		t.Fatal(err)
   193  	}
   194  }
   195  
   196  func TestMultiWrite_Progress(t *testing.T) {
   197  	idx, err := random.Index(1000, 3, 3)
   198  	if err != nil {
   199  		t.Fatal(err)
   200  	}
   201  	c := make(chan v1.Update, 1000)
   202  
   203  	// Set up a fake registry.
   204  	s := httptest.NewServer(registry.New())
   205  	defer s.Close()
   206  	u, err := url.Parse(s.URL)
   207  	if err != nil {
   208  		t.Fatal(err)
   209  	}
   210  	ref, err := name.ParseReference(fmt.Sprintf("%s/test/progress/upload", u.Host))
   211  	if err != nil {
   212  		t.Fatal(err)
   213  	}
   214  	ref2, err := name.ParseReference(fmt.Sprintf("%s/test/progress/upload:again", u.Host))
   215  	if err != nil {
   216  		t.Fatal(err)
   217  	}
   218  
   219  	if err := MultiWrite(map[name.Reference]Taggable{
   220  		ref:  idx,
   221  		ref2: idx,
   222  	}, WithProgress(c)); err != nil {
   223  		t.Fatalf("MultiWrite: %v", err)
   224  	}
   225  
   226  	if err := checkUpdates(c); err != nil {
   227  		t.Fatal(err)
   228  	}
   229  }
   230  
   231  func TestMultiWrite_Progress_Retry(t *testing.T) {
   232  	idx, err := random.Index(1000, 3, 3)
   233  	if err != nil {
   234  		t.Fatal(err)
   235  	}
   236  	c := make(chan v1.Update, 1000)
   237  
   238  	// Set up a fake registry.
   239  	handler := registry.New()
   240  	numOfInternalServerErrors := 0
   241  	var mu sync.Mutex
   242  	registryThatFailsOnFirstUpload := http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) {
   243  		mu.Lock()
   244  		defer mu.Unlock()
   245  		if strings.Contains(request.URL.Path, "/manifests/") && numOfInternalServerErrors < 1 {
   246  			numOfInternalServerErrors++
   247  			responseWriter.WriteHeader(500)
   248  			return
   249  		}
   250  		handler.ServeHTTP(responseWriter, request)
   251  	})
   252  
   253  	s := httptest.NewServer(registryThatFailsOnFirstUpload)
   254  	defer s.Close()
   255  	u, err := url.Parse(s.URL)
   256  	if err != nil {
   257  		t.Fatal(err)
   258  	}
   259  
   260  	ref, err := name.ParseReference(fmt.Sprintf("%s/test/progress/upload", u.Host))
   261  	if err != nil {
   262  		t.Fatal(err)
   263  	}
   264  	ref2, err := name.ParseReference(fmt.Sprintf("%s/test/progress/upload:again", u.Host))
   265  	if err != nil {
   266  		t.Fatal(err)
   267  	}
   268  
   269  	if err := MultiWrite(map[name.Reference]Taggable{
   270  		ref:  idx,
   271  		ref2: idx,
   272  	}, WithProgress(c), WithRetryBackoff(fastBackoff)); err != nil {
   273  		t.Fatalf("MultiWrite: %v", err)
   274  	}
   275  
   276  	if err := checkUpdates(c); err != nil {
   277  		t.Fatal(err)
   278  	}
   279  }
   280  
   281  func TestWriteLayer_Progress_Retry(t *testing.T) {
   282  	l, err := random.Layer(100000, types.OCIUncompressedLayer)
   283  	if err != nil {
   284  		t.Fatal(err)
   285  	}
   286  	c := make(chan v1.Update, 200)
   287  
   288  	// Set up a fake registry.
   289  	handler := registry.New()
   290  
   291  	numOfInternalServerErrors := 0
   292  	registryThatFailsOnFirstUpload := http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) {
   293  		if request.Method == http.MethodPatch && strings.Contains(request.URL.Path, "upload/blobs/uploads") && numOfInternalServerErrors < 1 {
   294  			numOfInternalServerErrors++
   295  			responseWriter.WriteHeader(500)
   296  			return
   297  		}
   298  		handler.ServeHTTP(responseWriter, request)
   299  	})
   300  
   301  	s := httptest.NewServer(registryThatFailsOnFirstUpload)
   302  	defer s.Close()
   303  	u, err := url.Parse(s.URL)
   304  	if err != nil {
   305  		t.Fatal(err)
   306  	}
   307  	dst := fmt.Sprintf("%s/test/progress/upload", u.Host)
   308  	ref, err := name.ParseReference(dst)
   309  	if err != nil {
   310  		t.Fatal(err)
   311  	}
   312  
   313  	if err := WriteLayer(ref.Context(), l, WithProgress(c), WithRetryBackoff(fastBackoff)); err != nil {
   314  		t.Fatalf("WriteLayer: %v", err)
   315  	}
   316  
   317  	everyUpdate := []v1.Update{}
   318  	for update := range c {
   319  		everyUpdate = append(everyUpdate, update)
   320  	}
   321  
   322  	if diff := cmp.Diff(everyUpdate, []v1.Update{
   323  		{Total: 101921, Complete: 32768},
   324  		{Total: 101921, Complete: 65536},
   325  		{Total: 101921, Complete: 98304},
   326  		{Total: 101921, Complete: 101921},
   327  		// retry results in the same messages sent to the updates channel
   328  		{Total: 101921, Complete: 0},
   329  		{Total: 101921, Complete: 32768},
   330  		{Total: 101921, Complete: 65536},
   331  		{Total: 101921, Complete: 98304},
   332  		{Total: 101921, Complete: 101921},
   333  	}); diff != "" {
   334  		t.Errorf("received updates (-want +got) = %s", diff)
   335  	}
   336  }
   337  
   338  func TestWriteLayer_Progress_Error(t *testing.T) {
   339  	l, err := random.Layer(100000, types.OCIUncompressedLayer)
   340  	if err != nil {
   341  		t.Fatal(err)
   342  	}
   343  	c := make(chan v1.Update, 200)
   344  
   345  	// Set up a fake registry.
   346  	handler := registry.New()
   347  	registryThatAlwaysFails := http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) {
   348  		if request.Method == http.MethodPatch && strings.Contains(request.URL.Path, "blobs/uploads") {
   349  			responseWriter.WriteHeader(403)
   350  		}
   351  		handler.ServeHTTP(responseWriter, request)
   352  	})
   353  
   354  	s := httptest.NewServer(registryThatAlwaysFails)
   355  	defer s.Close()
   356  	u, err := url.Parse(s.URL)
   357  	if err != nil {
   358  		t.Fatal(err)
   359  	}
   360  	dst := fmt.Sprintf("%s/test/progress/upload", u.Host)
   361  	ref, err := name.ParseReference(dst)
   362  	if err != nil {
   363  		t.Fatal(err)
   364  	}
   365  
   366  	if err := WriteLayer(ref.Context(), l, WithProgress(c)); err == nil {
   367  		t.Errorf("WriteLayer: wanted error, got nil")
   368  	}
   369  
   370  	everyUpdate := []v1.Update{}
   371  	for update := range c {
   372  		everyUpdate = append(everyUpdate, update)
   373  	}
   374  
   375  	if diff := cmp.Diff(everyUpdate[:len(everyUpdate)-1], []v1.Update{
   376  		{Total: 101921, Complete: 32768},
   377  		{Total: 101921, Complete: 65536},
   378  		{Total: 101921, Complete: 98304},
   379  		{Total: 101921, Complete: 101921},
   380  		// retry results in the same messages sent to the updates channel
   381  		{Total: 101921, Complete: 0},
   382  	}); diff != "" {
   383  		t.Errorf("received updates (-want +got) = %s", diff)
   384  	}
   385  	if everyUpdate[len(everyUpdate)-1].Error == nil {
   386  		t.Errorf("Last update had nil error")
   387  	}
   388  }
   389  
   390  func TestWrite_Progress_WithNonDistributableLayer_AndIncludeNonDistributableLayersOption(t *testing.T) {
   391  	ociLayer, err := random.Layer(1000, types.OCILayer)
   392  	if err != nil {
   393  		t.Fatal(err)
   394  	}
   395  
   396  	nonDistributableLayer, err := random.Layer(1000, types.OCIRestrictedLayer)
   397  	if err != nil {
   398  		t.Fatal(err)
   399  	}
   400  
   401  	img, err := mutate.AppendLayers(empty.Image, ociLayer, nonDistributableLayer)
   402  	if err != nil {
   403  		t.Fatal(err)
   404  	}
   405  
   406  	c := make(chan v1.Update, 200)
   407  
   408  	// Set up a fake registry.
   409  	s := httptest.NewServer(registry.New())
   410  	defer s.Close()
   411  	u, err := url.Parse(s.URL)
   412  	if err != nil {
   413  		t.Fatal(err)
   414  	}
   415  	dst := fmt.Sprintf("%s/test/progress/upload", u.Host)
   416  	ref, err := name.ParseReference(dst)
   417  	if err != nil {
   418  		t.Fatal(err)
   419  	}
   420  
   421  	if err := Write(ref, img, WithProgress(c), WithNondistributable); err != nil {
   422  		t.Fatalf("Write: %v", err)
   423  	}
   424  
   425  	if err := checkUpdates(c); err != nil {
   426  		t.Fatal(err)
   427  	}
   428  }
   429  
   430  // checkUpdates checks that updates show steady progress toward a total, and
   431  // don't describe errors.
   432  func checkUpdates(updates <-chan v1.Update) error {
   433  	var high, total int64
   434  	for u := range updates {
   435  		if u.Error != nil {
   436  			return u.Error
   437  		}
   438  
   439  		if u.Total < total {
   440  			return fmt.Errorf("total changed: was %d, saw %d", total, u.Total)
   441  		}
   442  
   443  		total = u.Total
   444  
   445  		if u.Complete < high {
   446  			return fmt.Errorf("saw progress revert: was high of %d, saw %d", high, u.Complete)
   447  		}
   448  		high = u.Complete
   449  	}
   450  
   451  	if high > total {
   452  		return fmt.Errorf("final progress (%d) exceeded total (%d) by %d", high, total, high-total)
   453  	} else if high < total {
   454  		return fmt.Errorf("final progress (%d) did not reach total (%d) by %d", high, total, total-high)
   455  	}
   456  
   457  	return nil
   458  }
   459
View as plain text