multi_write_test.go

Documentation: github.com/google/go-containerregistry/pkg/v1/remote

     1  // Copyright 2020 Google LLC All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //    http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package remote
    16  
    17  import (
    18  	"context"
    19  	"io"
    20  	"log"
    21  	"net/http"
    22  	"net/http/httptest"
    23  	"net/url"
    24  	"os"
    25  	"strings"
    26  	"testing"
    27  
    28  	"github.com/google/go-containerregistry/pkg/authn"
    29  	"github.com/google/go-containerregistry/pkg/logs"
    30  	"github.com/google/go-containerregistry/pkg/name"
    31  	"github.com/google/go-containerregistry/pkg/registry"
    32  	v1 "github.com/google/go-containerregistry/pkg/v1"
    33  	"github.com/google/go-containerregistry/pkg/v1/empty"
    34  	"github.com/google/go-containerregistry/pkg/v1/mutate"
    35  	"github.com/google/go-containerregistry/pkg/v1/random"
    36  	"github.com/google/go-containerregistry/pkg/v1/remote/transport"
    37  	"github.com/google/go-containerregistry/pkg/v1/stream"
    38  	"github.com/google/go-containerregistry/pkg/v1/types"
    39  	"github.com/google/go-containerregistry/pkg/v1/validate"
    40  )
    41  
    42  func streamable(t *testing.T) v1.Layer {
    43  	t.Helper()
    44  	rl, err := random.Layer(1024, types.OCIUncompressedLayer)
    45  	if err != nil {
    46  		t.Fatal("random.Layer:", err)
    47  	}
    48  	rc, err := rl.Uncompressed()
    49  	if err != nil {
    50  		t.Fatalf("Uncompressed(): %v", err)
    51  	}
    52  
    53  	return stream.NewLayer(rc)
    54  }
    55  
    56  type rawManifest struct {
    57  	b []byte
    58  }
    59  
    60  func (r *rawManifest) RawManifest() ([]byte, error) {
    61  	return r.b, nil
    62  }
    63  
    64  func TestMultiWrite(t *testing.T) {
    65  	c := make(chan v1.Update, 1000)
    66  
    67  	logs.Progress.SetOutput(os.Stderr)
    68  	logs.Warn.SetOutput(os.Stderr)
    69  
    70  	// Create a random image.
    71  	img1, err := random.Image(1024, 2)
    72  	if err != nil {
    73  		t.Fatal("random.Image:", err)
    74  	}
    75  
    76  	// Create another image that's based on the first.
    77  	rl, err := random.Layer(1024, types.OCIUncompressedLayer)
    78  	if err != nil {
    79  		t.Fatal("random.Layer:", err)
    80  	}
    81  	img2, err := mutate.AppendLayers(img1, rl, streamable(t))
    82  	if err != nil {
    83  		t.Fatal("mutate.AppendLayers:", err)
    84  	}
    85  
    86  	// Also create a random index of images.
    87  	subidx, err := random.Index(1024, 2, 3)
    88  	if err != nil {
    89  		t.Fatal("random.Index:", err)
    90  	}
    91  
    92  	// Add a sub-sub-index of random images.
    93  	subsubidx, err := random.Index(1024, 3, 4)
    94  	if err != nil {
    95  		t.Fatal("random.Index:", err)
    96  	}
    97  	subidx = mutate.AppendManifests(subidx, mutate.IndexAddendum{Add: subsubidx})
    98  
    99  	// Create an index containing both images and the index above.
   100  	idx := mutate.AppendManifests(empty.Index,
   101  		mutate.IndexAddendum{Add: img1},
   102  		mutate.IndexAddendum{Add: img2},
   103  		mutate.IndexAddendum{Add: subidx},
   104  		mutate.IndexAddendum{Add: rl},
   105  	)
   106  
   107  	// Set up a fake registry.
   108  	nopLog := log.New(io.Discard, "", 0)
   109  	s := httptest.NewServer(registry.New(registry.Logger(nopLog)))
   110  	defer s.Close()
   111  	u, err := url.Parse(s.URL)
   112  	if err != nil {
   113  		t.Fatal(err)
   114  	}
   115  
   116  	// Write both images and the manifest list.
   117  	tag1, tag2, tag3, tag4, tag5 := mustNewTag(t, u.Host+"/repo2:tag1"), mustNewTag(t, u.Host+"/repo:tag2"), mustNewTag(t, u.Host+"/repo:tag3"), mustNewTag(t, u.Host+"/repo:tag4"), mustNewTag(t, u.Host+"/repo1:tag4")
   118  
   119  	if err := MultiWrite(map[name.Reference]Taggable{
   120  		tag1: img1,
   121  		tag2: img2,
   122  		tag3: idx,
   123  	}, WithProgress(c)); err != nil {
   124  		t.Fatal("MultiWrite:", err)
   125  	}
   126  
   127  	// Check that tagged images are present.
   128  	for _, tag := range []name.Tag{tag1, tag2} {
   129  		got, err := Image(tag)
   130  		if err != nil {
   131  			t.Error(err)
   132  			continue
   133  		}
   134  		if err := validate.Image(got); err != nil {
   135  			t.Error("Validate() =", err)
   136  		}
   137  	}
   138  
   139  	// Check that tagged manfest list is present and valid.
   140  	got, err := Index(tag3)
   141  	if err != nil {
   142  		t.Fatal(err)
   143  	}
   144  	if err := validate.Index(got); err != nil {
   145  		t.Error("Validate() =", err)
   146  	}
   147  
   148  	if err := checkUpdates(c); err != nil {
   149  		t.Fatal(err)
   150  	}
   151  
   152  	desc1, err := Get(tag1)
   153  	if err != nil {
   154  		t.Fatal(err)
   155  	}
   156  	desc2, err := Get(tag3)
   157  	if err != nil {
   158  		t.Fatal(err)
   159  	}
   160  
   161  	rm := &rawManifest{[]byte("{}")}
   162  
   163  	// Hit "already exists" coverage paths and move some tags.
   164  	if err := MultiWrite(map[name.Reference]Taggable{
   165  		tag1: img2,
   166  		tag2: img1,
   167  		tag3: desc2,
   168  		tag4: desc1,
   169  		tag5: rm,
   170  	}); err != nil {
   171  		t.Fatal("MultiWrite:", err)
   172  	}
   173  }
   174  
   175  func TestMultiWriteWithNondistributableLayer(t *testing.T) {
   176  	// Create a random image.
   177  	img1, err := random.Image(1024, 2)
   178  	if err != nil {
   179  		t.Fatal("random.Image:", err)
   180  	}
   181  
   182  	// Create another image that's based on the first.
   183  	rl, err := random.Layer(1024, types.OCIRestrictedLayer)
   184  	if err != nil {
   185  		t.Fatal("random.Layer:", err)
   186  	}
   187  	img, err := mutate.AppendLayers(img1, rl)
   188  	if err != nil {
   189  		t.Fatal("mutate.AppendLayers:", err)
   190  	}
   191  
   192  	// Set up a fake registry.
   193  	s := httptest.NewServer(registry.New())
   194  	defer s.Close()
   195  	u, err := url.Parse(s.URL)
   196  	if err != nil {
   197  		t.Fatal(err)
   198  	}
   199  
   200  	// Write the image.
   201  	tag1 := mustNewTag(t, u.Host+"/repo:tag1")
   202  	if err := MultiWrite(map[name.Reference]Taggable{tag1: img}, WithNondistributable); err != nil {
   203  		t.Error("Write:", err)
   204  	}
   205  
   206  	// Check that tagged image is present.
   207  	got, err := Image(tag1)
   208  	if err != nil {
   209  		t.Error(err)
   210  	}
   211  	if err := validate.Image(got); err != nil {
   212  		t.Error("Validate() =", err)
   213  	}
   214  }
   215  
   216  func TestMultiWrite_Retry(t *testing.T) {
   217  	// Create a random image.
   218  	img1, err := random.Image(1024, 2)
   219  	if err != nil {
   220  		t.Fatal("random.Image:", err)
   221  	}
   222  
   223  	t.Run("retry http error 500", func(t *testing.T) {
   224  		// Set up a fake registry.
   225  		handler := registry.New()
   226  
   227  		numOfInternalServerErrors := 0
   228  		registryThatFailsOnFirstUpload := http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) {
   229  			if strings.Contains(request.URL.Path, "/manifests/") && numOfInternalServerErrors < 1 {
   230  				numOfInternalServerErrors++
   231  				responseWriter.WriteHeader(500)
   232  				return
   233  			}
   234  			handler.ServeHTTP(responseWriter, request)
   235  		})
   236  
   237  		s := httptest.NewServer(registryThatFailsOnFirstUpload)
   238  		defer s.Close()
   239  		u, err := url.Parse(s.URL)
   240  		if err != nil {
   241  			t.Fatal(err)
   242  		}
   243  
   244  		tag1 := mustNewTag(t, u.Host+"/repo:tag1")
   245  		if err := MultiWrite(map[name.Reference]Taggable{
   246  			tag1: img1,
   247  		}, WithRetryBackoff(fastBackoff)); err != nil {
   248  			t.Error("Write:", err)
   249  		}
   250  	})
   251  
   252  	t.Run("do not retry transport errors if transport.Wrapper is used", func(t *testing.T) {
   253  		// reference a http server that is not listening (used to pick a port that isn't listening)
   254  		onlyHandlesPing := http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) {
   255  			if strings.HasSuffix(request.URL.Path, "/v2/") {
   256  				responseWriter.WriteHeader(200)
   257  				return
   258  			}
   259  		})
   260  		s := httptest.NewServer(onlyHandlesPing)
   261  		defer s.Close()
   262  
   263  		u, err := url.Parse(s.URL)
   264  		if err != nil {
   265  			t.Fatal(err)
   266  		}
   267  
   268  		tag1 := mustNewTag(t, u.Host+"/repo:tag1")
   269  
   270  		// using a transport.Wrapper, meaning retry logic should not be wrapped
   271  		doesNotRetryTransport := &countTransport{inner: http.DefaultTransport}
   272  		transportWrapper, err := transport.NewWithContext(context.Background(), tag1.Repository.Registry, authn.Anonymous, doesNotRetryTransport, nil)
   273  		if err != nil {
   274  			t.Fatal(err)
   275  		}
   276  
   277  		noRetry := func(error) bool { return false }
   278  
   279  		if err := MultiWrite(map[name.Reference]Taggable{
   280  			tag1: img1,
   281  		}, WithTransport(transportWrapper), WithJobs(1), WithRetryPredicate(noRetry)); err == nil {
   282  			t.Errorf("Expected an error, got nil")
   283  		}
   284  
   285  		// expect count == 1 since jobs is set to 1 and we should not retry on transport eof error
   286  		if doesNotRetryTransport.count != 1 {
   287  			t.Errorf("Incorrect count, got %d, want %d", doesNotRetryTransport.count, 1)
   288  		}
   289  	})
   290  
   291  	t.Run("do not add UserAgent if transport.Wrapper is used", func(t *testing.T) {
   292  		expectedNotUsedUserAgent := "TEST_USER_AGENT"
   293  
   294  		handler := registry.New()
   295  
   296  		registryThatAssertsUserAgentIsCorrect := http.HandlerFunc(func(responseWriter http.ResponseWriter, request *http.Request) {
   297  			if strings.Contains(request.Header.Get("User-Agent"), expectedNotUsedUserAgent) {
   298  				t.Fatalf("Should not contain User-Agent: %s, Got: %s", expectedNotUsedUserAgent, request.Header.Get("User-Agent"))
   299  			}
   300  
   301  			handler.ServeHTTP(responseWriter, request)
   302  		})
   303  
   304  		s := httptest.NewServer(registryThatAssertsUserAgentIsCorrect)
   305  
   306  		defer s.Close()
   307  		u, err := url.Parse(s.URL)
   308  		if err != nil {
   309  			t.Fatal(err)
   310  		}
   311  
   312  		tag1 := mustNewTag(t, u.Host+"/repo:tag1")
   313  		// using a transport.Wrapper, meaning retry logic should not be wrapped
   314  		transportWrapper, err := transport.NewWithContext(context.Background(), tag1.Repository.Registry, authn.Anonymous, http.DefaultTransport, nil)
   315  		if err != nil {
   316  			t.Fatal(err)
   317  		}
   318  
   319  		if err := MultiWrite(map[name.Reference]Taggable{
   320  			tag1: img1,
   321  		}, WithTransport(transportWrapper), WithUserAgent(expectedNotUsedUserAgent)); err != nil {
   322  			t.Fatal(err)
   323  		}
   324  	})
   325  }
   326  
   327  // TestMultiWrite_Deep tests that a deeply nested tree of manifest lists gets
   328  // pushed in the correct order (i.e., each level in sequence).
   329  func TestMultiWrite_Deep(t *testing.T) {
   330  	idx, err := random.Index(1024, 2, 2)
   331  	if err != nil {
   332  		t.Fatal("random.Image:", err)
   333  	}
   334  	for i := 0; i < 4; i++ {
   335  		idx = mutate.AppendManifests(idx, mutate.IndexAddendum{Add: idx})
   336  	}
   337  
   338  	// Set up a fake registry (with NOP logger to avoid spamming test logs).
   339  	nopLog := log.New(io.Discard, "", 0)
   340  	s := httptest.NewServer(registry.New(registry.Logger(nopLog)))
   341  	defer s.Close()
   342  	u, err := url.Parse(s.URL)
   343  	if err != nil {
   344  		t.Fatal(err)
   345  	}
   346  
   347  	// Write both images and the manifest list.
   348  	tag := mustNewTag(t, u.Host+"/repo:tag")
   349  	if err := MultiWrite(map[name.Reference]Taggable{
   350  		tag: idx,
   351  	}); err != nil {
   352  		t.Error("Write:", err)
   353  	}
   354  
   355  	// Check that tagged manfest list is present and valid.
   356  	got, err := Index(tag)
   357  	if err != nil {
   358  		t.Fatal(err)
   359  	}
   360  	if err := validate.Index(got); err != nil {
   361  		t.Error("Validate() =", err)
   362  	}
   363  }
   364  
   365  type countTransport struct {
   366  	count int
   367  	inner http.RoundTripper
   368  }
   369  
   370  func (t *countTransport) RoundTrip(req *http.Request) (*http.Response, error) {
   371  	if strings.HasSuffix(req.URL.Path, "/v2/") {
   372  		return t.inner.RoundTrip(req)
   373  	}
   374  
   375  	t.count++
   376  	return nil, io.ErrUnexpectedEOF
   377  }
   378
View as plain text