fetcher.go

Documentation: github.com/google/go-containerregistry/pkg/v1/remote

     1  // Copyright 2023 Google LLC All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //    http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package remote
    16  
    17  import (
    18  	"bytes"
    19  	"context"
    20  	"fmt"
    21  	"io"
    22  	"net/http"
    23  	"net/url"
    24  	"strings"
    25  
    26  	"github.com/google/go-containerregistry/internal/redact"
    27  	"github.com/google/go-containerregistry/internal/verify"
    28  	"github.com/google/go-containerregistry/pkg/authn"
    29  	"github.com/google/go-containerregistry/pkg/name"
    30  	v1 "github.com/google/go-containerregistry/pkg/v1"
    31  	"github.com/google/go-containerregistry/pkg/v1/remote/transport"
    32  	"github.com/google/go-containerregistry/pkg/v1/types"
    33  )
    34  
    35  const (
    36  	kib           = 1024
    37  	mib           = 1024 * kib
    38  	manifestLimit = 100 * mib
    39  )
    40  
    41  // fetcher implements methods for reading from a registry.
    42  type fetcher struct {
    43  	target resource
    44  	client *http.Client
    45  }
    46  
    47  func makeFetcher(ctx context.Context, target resource, o *options) (*fetcher, error) {
    48  	auth := o.auth
    49  	if o.keychain != nil {
    50  		kauth, err := o.keychain.Resolve(target)
    51  		if err != nil {
    52  			return nil, err
    53  		}
    54  		auth = kauth
    55  	}
    56  
    57  	reg, ok := target.(name.Registry)
    58  	if !ok {
    59  		repo, ok := target.(name.Repository)
    60  		if !ok {
    61  			return nil, fmt.Errorf("unexpected resource: %T", target)
    62  		}
    63  		reg = repo.Registry
    64  	}
    65  
    66  	tr, err := transport.NewWithContext(ctx, reg, auth, o.transport, []string{target.Scope(transport.PullScope)})
    67  	if err != nil {
    68  		return nil, err
    69  	}
    70  	return &fetcher{
    71  		target: target,
    72  		client: &http.Client{Transport: tr},
    73  	}, nil
    74  }
    75  
    76  func (f *fetcher) Do(req *http.Request) (*http.Response, error) {
    77  	return f.client.Do(req)
    78  }
    79  
    80  type resource interface {
    81  	Scheme() string
    82  	RegistryStr() string
    83  	Scope(string) string
    84  
    85  	authn.Resource
    86  }
    87  
    88  // url returns a url.Url for the specified path in the context of this remote image reference.
    89  func (f *fetcher) url(resource, identifier string) url.URL {
    90  	u := url.URL{
    91  		Scheme: f.target.Scheme(),
    92  		Host:   f.target.RegistryStr(),
    93  		// Default path if this is not a repository.
    94  		Path: "/v2/_catalog",
    95  	}
    96  	if repo, ok := f.target.(name.Repository); ok {
    97  		u.Path = fmt.Sprintf("/v2/%s/%s/%s", repo.RepositoryStr(), resource, identifier)
    98  	}
    99  	return u
   100  }
   101  
   102  func (f *fetcher) get(ctx context.Context, ref name.Reference, acceptable []types.MediaType, platform v1.Platform) (*Descriptor, error) {
   103  	b, desc, err := f.fetchManifest(ctx, ref, acceptable)
   104  	if err != nil {
   105  		return nil, err
   106  	}
   107  	return &Descriptor{
   108  		ref:        ref,
   109  		ctx:        ctx,
   110  		fetcher:    *f,
   111  		Manifest:   b,
   112  		Descriptor: *desc,
   113  		platform:   platform,
   114  	}, nil
   115  }
   116  
   117  func (f *fetcher) fetchManifest(ctx context.Context, ref name.Reference, acceptable []types.MediaType) ([]byte, *v1.Descriptor, error) {
   118  	u := f.url("manifests", ref.Identifier())
   119  	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
   120  	if err != nil {
   121  		return nil, nil, err
   122  	}
   123  	accept := []string{}
   124  	for _, mt := range acceptable {
   125  		accept = append(accept, string(mt))
   126  	}
   127  	req.Header.Set("Accept", strings.Join(accept, ","))
   128  
   129  	resp, err := f.client.Do(req.WithContext(ctx))
   130  	if err != nil {
   131  		return nil, nil, err
   132  	}
   133  	defer resp.Body.Close()
   134  
   135  	if err := transport.CheckError(resp, http.StatusOK); err != nil {
   136  		return nil, nil, err
   137  	}
   138  
   139  	manifest, err := io.ReadAll(io.LimitReader(resp.Body, manifestLimit))
   140  	if err != nil {
   141  		return nil, nil, err
   142  	}
   143  
   144  	digest, size, err := v1.SHA256(bytes.NewReader(manifest))
   145  	if err != nil {
   146  		return nil, nil, err
   147  	}
   148  
   149  	mediaType := types.MediaType(resp.Header.Get("Content-Type"))
   150  	contentDigest, err := v1.NewHash(resp.Header.Get("Docker-Content-Digest"))
   151  	if err == nil && mediaType == types.DockerManifestSchema1Signed {
   152  		// If we can parse the digest from the header, and it's a signed schema 1
   153  		// manifest, let's use that for the digest to appease older registries.
   154  		digest = contentDigest
   155  	}
   156  
   157  	// Validate the digest matches what we asked for, if pulling by digest.
   158  	if dgst, ok := ref.(name.Digest); ok {
   159  		if digest.String() != dgst.DigestStr() {
   160  			return nil, nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), ref)
   161  		}
   162  	}
   163  
   164  	var artifactType string
   165  	mf, _ := v1.ParseManifest(bytes.NewReader(manifest))
   166  	// Failing to parse as a manifest should just be ignored.
   167  	// The manifest might not be valid, and that's okay.
   168  	if mf != nil && !mf.Config.MediaType.IsConfig() {
   169  		artifactType = string(mf.Config.MediaType)
   170  	}
   171  
   172  	// Do nothing for tags; I give up.
   173  	//
   174  	// We'd like to validate that the "Docker-Content-Digest" header matches what is returned by the registry,
   175  	// but so many registries implement this incorrectly that it's not worth checking.
   176  	//
   177  	// For reference:
   178  	// https://github.com/GoogleContainerTools/kaniko/issues/298
   179  
   180  	// Return all this info since we have to calculate it anyway.
   181  	desc := v1.Descriptor{
   182  		Digest:       digest,
   183  		Size:         size,
   184  		MediaType:    mediaType,
   185  		ArtifactType: artifactType,
   186  	}
   187  
   188  	return manifest, &desc, nil
   189  }
   190  
   191  func (f *fetcher) headManifest(ctx context.Context, ref name.Reference, acceptable []types.MediaType) (*v1.Descriptor, error) {
   192  	u := f.url("manifests", ref.Identifier())
   193  	req, err := http.NewRequest(http.MethodHead, u.String(), nil)
   194  	if err != nil {
   195  		return nil, err
   196  	}
   197  	accept := []string{}
   198  	for _, mt := range acceptable {
   199  		accept = append(accept, string(mt))
   200  	}
   201  	req.Header.Set("Accept", strings.Join(accept, ","))
   202  
   203  	resp, err := f.client.Do(req.WithContext(ctx))
   204  	if err != nil {
   205  		return nil, err
   206  	}
   207  	defer resp.Body.Close()
   208  
   209  	if err := transport.CheckError(resp, http.StatusOK); err != nil {
   210  		return nil, err
   211  	}
   212  
   213  	mth := resp.Header.Get("Content-Type")
   214  	if mth == "" {
   215  		return nil, fmt.Errorf("HEAD %s: response did not include Content-Type header", u.String())
   216  	}
   217  	mediaType := types.MediaType(mth)
   218  
   219  	size := resp.ContentLength
   220  	if size == -1 {
   221  		return nil, fmt.Errorf("GET %s: response did not include Content-Length header", u.String())
   222  	}
   223  
   224  	dh := resp.Header.Get("Docker-Content-Digest")
   225  	if dh == "" {
   226  		return nil, fmt.Errorf("HEAD %s: response did not include Docker-Content-Digest header", u.String())
   227  	}
   228  	digest, err := v1.NewHash(dh)
   229  	if err != nil {
   230  		return nil, err
   231  	}
   232  
   233  	// Validate the digest matches what we asked for, if pulling by digest.
   234  	if dgst, ok := ref.(name.Digest); ok {
   235  		if digest.String() != dgst.DigestStr() {
   236  			return nil, fmt.Errorf("manifest digest: %q does not match requested digest: %q for %q", digest, dgst.DigestStr(), ref)
   237  		}
   238  	}
   239  
   240  	// Return all this info since we have to calculate it anyway.
   241  	return &v1.Descriptor{
   242  		Digest:    digest,
   243  		Size:      size,
   244  		MediaType: mediaType,
   245  	}, nil
   246  }
   247  
   248  func (f *fetcher) fetchBlob(ctx context.Context, size int64, h v1.Hash) (io.ReadCloser, error) {
   249  	u := f.url("blobs", h.String())
   250  	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
   251  	if err != nil {
   252  		return nil, err
   253  	}
   254  
   255  	resp, err := f.client.Do(req.WithContext(ctx))
   256  	if err != nil {
   257  		return nil, redact.Error(err)
   258  	}
   259  
   260  	if err := transport.CheckError(resp, http.StatusOK); err != nil {
   261  		resp.Body.Close()
   262  		return nil, err
   263  	}
   264  
   265  	// Do whatever we can.
   266  	// If we have an expected size and Content-Length doesn't match, return an error.
   267  	// If we don't have an expected size and we do have a Content-Length, use Content-Length.
   268  	if hsize := resp.ContentLength; hsize != -1 {
   269  		if size == verify.SizeUnknown {
   270  			size = hsize
   271  		} else if hsize != size {
   272  			return nil, fmt.Errorf("GET %s: Content-Length header %d does not match expected size %d", u.String(), hsize, size)
   273  		}
   274  	}
   275  
   276  	return verify.ReadCloser(resp.Body, size, h)
   277  }
   278  
   279  func (f *fetcher) headBlob(ctx context.Context, h v1.Hash) (*http.Response, error) {
   280  	u := f.url("blobs", h.String())
   281  	req, err := http.NewRequest(http.MethodHead, u.String(), nil)
   282  	if err != nil {
   283  		return nil, err
   284  	}
   285  
   286  	resp, err := f.client.Do(req.WithContext(ctx))
   287  	if err != nil {
   288  		return nil, redact.Error(err)
   289  	}
   290  
   291  	if err := transport.CheckError(resp, http.StatusOK); err != nil {
   292  		resp.Body.Close()
   293  		return nil, err
   294  	}
   295  
   296  	return resp, nil
   297  }
   298  
   299  func (f *fetcher) blobExists(ctx context.Context, h v1.Hash) (bool, error) {
   300  	u := f.url("blobs", h.String())
   301  	req, err := http.NewRequest(http.MethodHead, u.String(), nil)
   302  	if err != nil {
   303  		return false, err
   304  	}
   305  
   306  	resp, err := f.client.Do(req.WithContext(ctx))
   307  	if err != nil {
   308  		return false, redact.Error(err)
   309  	}
   310  	defer resp.Body.Close()
   311  
   312  	if err := transport.CheckError(resp, http.StatusOK, http.StatusNotFound); err != nil {
   313  		return false, err
   314  	}
   315  
   316  	return resp.StatusCode == http.StatusOK, nil
   317  }
   318
View as plain text