client.go

Documentation: cuelang.org/go/mod/modregistry

     1  // Copyright 2023 CUE Authors
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  // Package modregistry provides functionality for reading and writing
    16  // CUE modules from an OCI registry.
    17  //
    18  // WARNING: THIS PACKAGE IS EXPERIMENTAL.
    19  // ITS API MAY CHANGE AT ANY TIME.
    20  package modregistry
    21  
    22  import (
    23  	"archive/zip"
    24  	"bytes"
    25  	"context"
    26  	"encoding/json"
    27  	"errors"
    28  	"fmt"
    29  	"io"
    30  	"strings"
    31  
    32  	"cuelabs.dev/go/oci/ociregistry"
    33  	"cuelang.org/go/internal/mod/semver"
    34  	digest "github.com/opencontainers/go-digest"
    35  	specs "github.com/opencontainers/image-spec/specs-go"
    36  	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
    37  
    38  	"cuelang.org/go/mod/modfile"
    39  	"cuelang.org/go/mod/module"
    40  	"cuelang.org/go/mod/modzip"
    41  )
    42  
    43  var ErrNotFound = fmt.Errorf("module not found")
    44  
    45  // Client represents a OCI-registry-backed client that
    46  // provides a store for CUE modules.
    47  type Client struct {
    48  	resolver Resolver
    49  }
    50  
    51  // Resolver resolves module paths to a registry and a location
    52  // within that registry.
    53  type Resolver interface {
    54  	// ResolveToRegistry resolves a base module path (without a version)
    55  	// and optional version to the location for that path.
    56  	//
    57  	// If the version is empty, the Tag in the returned Location
    58  	// will hold the prefix that all versions of the module in its
    59  	// repository have. That prefix will be followed by the version
    60  	// itself.
    61  	ResolveToRegistry(mpath, vers string) (RegistryLocation, error)
    62  }
    63  
    64  // RegistryLocation holds a registry and a location within it
    65  // that a specific module (or set of versions for a module)
    66  // will be stored.
    67  type RegistryLocation struct {
    68  	// Registry holds the registry to use to access the module.
    69  	Registry ociregistry.Interface
    70  	// Repository holds the repository where the module is located.
    71  	Repository string
    72  	// Tag holds the tag for the module version. If an empty version
    73  	// was passed to Resolve, it holds the prefix shared by all
    74  	// version tags for the module.
    75  	Tag string
    76  }
    77  
    78  const (
    79  	moduleArtifactType  = "application/vnd.cue.module.v1+json"
    80  	moduleFileMediaType = "application/vnd.cue.modulefile.v1"
    81  	moduleAnnotation    = "works.cue.module"
    82  )
    83  
    84  // NewClient returns a new client that talks to the registry at the given
    85  // hostname.
    86  func NewClient(registry ociregistry.Interface) *Client {
    87  	return &Client{
    88  		resolver: singleResolver{registry},
    89  	}
    90  }
    91  
    92  // NewClientWithResolver returns a new client that uses the given
    93  // resolver to decide which registries to fetch from or push to.
    94  func NewClientWithResolver(resolver Resolver) *Client {
    95  	return &Client{
    96  		resolver: resolver,
    97  	}
    98  }
    99  
   100  // GetModule returns the module instance for the given version.
   101  // It returns an error that satisfies errors.Is(ErrNotFound) if the
   102  // module is not present in the store at this version.
   103  func (c *Client) GetModule(ctx context.Context, m module.Version) (*Module, error) {
   104  	loc, err := c.resolve(m)
   105  	if err != nil {
   106  		return nil, err
   107  	}
   108  	rd, err := loc.Registry.GetTag(ctx, loc.Repository, loc.Tag)
   109  	if err != nil {
   110  		if errors.Is(err, ociregistry.ErrManifestUnknown) {
   111  			return nil, fmt.Errorf("module %v: %w", m, ErrNotFound)
   112  		}
   113  		return nil, fmt.Errorf("module %v: %v", m, err)
   114  	}
   115  	defer rd.Close()
   116  	data, err := io.ReadAll(rd)
   117  	if err != nil {
   118  		return nil, err
   119  	}
   120  
   121  	return c.GetModuleWithManifest(ctx, m, data, rd.Descriptor().MediaType)
   122  }
   123  
   124  // GetModuleWithManifest returns a module instance given
   125  // the top level manifest contents, without querying its tag.
   126  // It assumes that the module will be tagged with the given
   127  // version.
   128  func (c *Client) GetModuleWithManifest(ctx context.Context, m module.Version, contents []byte, mediaType string) (*Module, error) {
   129  	loc, err := c.resolve(m)
   130  	if err != nil {
   131  		return nil, err
   132  	}
   133  
   134  	manifest, err := unmarshalManifest(ctx, contents, mediaType)
   135  	if err != nil {
   136  		return nil, fmt.Errorf("module %v: %v", m, err)
   137  	}
   138  	if !isModule(manifest) {
   139  		return nil, fmt.Errorf("%v does not resolve to a manifest (media type is %q)", m, mediaType)
   140  	}
   141  	// TODO check type of manifest too.
   142  	if n := len(manifest.Layers); n != 2 {
   143  		return nil, fmt.Errorf("module manifest should refer to exactly two blobs, but got %d", n)
   144  	}
   145  	if !isModuleFile(manifest.Layers[1]) {
   146  		return nil, fmt.Errorf("unexpected media type %q for module file blob", manifest.Layers[1].MediaType)
   147  	}
   148  	// TODO check that the other blobs are of the expected type (application/zip).
   149  	return &Module{
   150  		client:         c,
   151  		loc:            loc,
   152  		version:        m,
   153  		manifest:       *manifest,
   154  		manifestDigest: digest.FromBytes(contents),
   155  	}, nil
   156  }
   157  
   158  // ModuleVersions returns all the versions for the module with the given path
   159  // sorted in semver order.
   160  // If m has a major version suffix, only versions with that major version will
   161  // be returned.
   162  func (c *Client) ModuleVersions(ctx context.Context, m string) ([]string, error) {
   163  	mpath, major, hasMajor := module.SplitPathVersion(m)
   164  	if !hasMajor {
   165  		mpath = m
   166  	}
   167  	loc, err := c.resolver.ResolveToRegistry(mpath, "")
   168  	if err != nil {
   169  		return nil, err
   170  	}
   171  	versions := []string{}
   172  	// Note: do not use c.repoName because that always expects
   173  	// a module path with a major version.
   174  	iter := loc.Registry.Tags(ctx, loc.Repository, "")
   175  	var _err error
   176  	iter(func(tag string, err error) bool {
   177  		if err != nil {
   178  			_err = err
   179  			return false
   180  		}
   181  		vers, ok := strings.CutPrefix(tag, loc.Tag)
   182  		if !ok || !semver.IsValid(vers) {
   183  			return true
   184  		}
   185  		if !hasMajor || semver.Major(vers) == major {
   186  			versions = append(versions, vers)
   187  		}
   188  		return true
   189  	})
   190  	if _err != nil && !isNotExist(_err) {
   191  		return nil, _err
   192  	}
   193  	semver.Sort(versions)
   194  	return versions, nil
   195  }
   196  
   197  // checkedModule represents module content that has passed the same
   198  // checks made by [Client.PutModule]. The caller should not mutate
   199  // any of the values returned by its methods.
   200  type checkedModule struct {
   201  	mv             module.Version
   202  	blobr          io.ReaderAt
   203  	size           int64
   204  	zipr           *zip.Reader
   205  	modFile        *modfile.File
   206  	modFileContent []byte
   207  }
   208  
   209  // putCheckedModule is like [Client.PutModule] except that it allows the
   210  // caller to do some additional checks (see [CheckModule] for more info).
   211  func (c *Client) putCheckedModule(ctx context.Context, m *checkedModule) error {
   212  	loc, err := c.resolve(m.mv)
   213  	if err != nil {
   214  		return err
   215  	}
   216  	selfDigest, err := digest.FromReader(io.NewSectionReader(m.blobr, 0, m.size))
   217  	if err != nil {
   218  		return fmt.Errorf("cannot read module zip file: %v", err)
   219  	}
   220  	// Upload the actual module's content
   221  	// TODO should we use a custom media type for this?
   222  	configDesc, err := c.scratchConfig(ctx, loc, moduleArtifactType)
   223  	if err != nil {
   224  		return fmt.Errorf("cannot make scratch config: %v", err)
   225  	}
   226  	manifest := &ocispec.Manifest{
   227  		Versioned: specs.Versioned{
   228  			SchemaVersion: 2, // historical value. does not pertain to OCI or docker version
   229  		},
   230  		MediaType: ocispec.MediaTypeImageManifest,
   231  		Config:    configDesc,
   232  		// One for self, one for module file.
   233  		Layers: []ocispec.Descriptor{{
   234  			Digest:    selfDigest,
   235  			MediaType: "application/zip",
   236  			Size:      m.size,
   237  		}, {
   238  			Digest:    digest.FromBytes(m.modFileContent),
   239  			MediaType: moduleFileMediaType,
   240  			Size:      int64(len(m.modFileContent)),
   241  		}},
   242  	}
   243  
   244  	if _, err := loc.Registry.PushBlob(ctx, loc.Repository, manifest.Layers[0], io.NewSectionReader(m.blobr, 0, m.size)); err != nil {
   245  		return fmt.Errorf("cannot push module contents: %v", err)
   246  	}
   247  	if _, err := loc.Registry.PushBlob(ctx, loc.Repository, manifest.Layers[1], bytes.NewReader(m.modFileContent)); err != nil {
   248  		return fmt.Errorf("cannot push cue.mod/module.cue contents: %v", err)
   249  	}
   250  	manifestData, err := json.Marshal(manifest)
   251  	if err != nil {
   252  		return fmt.Errorf("cannot marshal manifest: %v", err)
   253  	}
   254  	if _, err := loc.Registry.PushManifest(ctx, loc.Repository, loc.Tag, manifestData, ocispec.MediaTypeImageManifest); err != nil {
   255  		return fmt.Errorf("cannot tag %v: %v", m.mv, err)
   256  	}
   257  	return nil
   258  }
   259  
   260  // PutModule puts a module whose contents are held as a zip archive inside f.
   261  // It assumes all the module dependencies are correctly resolved and present
   262  // inside the cue.mod/module.cue file.
   263  //
   264  // TODO check deps are resolved correctly? Or is that too domain-specific for this package?
   265  // Is it a problem to call zip.CheckZip twice?
   266  func (c *Client) PutModule(ctx context.Context, m module.Version, r io.ReaderAt, size int64) error {
   267  	cm, err := checkModule(m, r, size)
   268  	if err != nil {
   269  		return err
   270  	}
   271  	return c.putCheckedModule(ctx, cm)
   272  }
   273  
   274  // checkModule checks a module's zip file before uploading it.
   275  // This does the same checks that [Client.PutModule] does, so
   276  // can be used to avoid doing duplicate work when an uploader
   277  // wishes to do more checks that are implemented by that method.
   278  //
   279  // Note that the returned [CheckedModule] value contains r, so will
   280  // be invalidated if r is closed.
   281  func checkModule(m module.Version, blobr io.ReaderAt, size int64) (*checkedModule, error) {
   282  	zipr, modf, _, err := modzip.CheckZip(m, blobr, size)
   283  	if err != nil {
   284  		return nil, fmt.Errorf("module zip file check failed: %v", err)
   285  	}
   286  	modFileContent, mf, err := checkModFile(m, modf)
   287  	if err != nil {
   288  		return nil, fmt.Errorf("module.cue file check failed: %v", err)
   289  	}
   290  	return &checkedModule{
   291  		mv:             m,
   292  		blobr:          blobr,
   293  		size:           size,
   294  		zipr:           zipr,
   295  		modFile:        mf,
   296  		modFileContent: modFileContent,
   297  	}, nil
   298  }
   299  
   300  func checkModFile(m module.Version, f *zip.File) ([]byte, *modfile.File, error) {
   301  	r, err := f.Open()
   302  	if err != nil {
   303  		return nil, nil, err
   304  	}
   305  	defer r.Close()
   306  	// TODO check max size?
   307  	data, err := io.ReadAll(r)
   308  	if err != nil {
   309  		return nil, nil, err
   310  	}
   311  	mf, err := modfile.Parse(data, f.Name)
   312  	if err != nil {
   313  		return nil, nil, err
   314  	}
   315  	if mf.Module != m.Path() {
   316  		return nil, nil, fmt.Errorf("module path %q found in %s does not match module path being published %q", mf.Module, f.Name, m.Path())
   317  	}
   318  	_, major, ok := module.SplitPathVersion(mf.Module)
   319  	if !ok {
   320  		// Note: can't happen because we already know that mf.Module is the same
   321  		// as m.Path which is a valid module path.
   322  		return nil, nil, fmt.Errorf("invalid module path %q", mf.Module)
   323  	}
   324  	wantMajor := semver.Major(m.Version())
   325  	if major != wantMajor {
   326  		// This can't actually happen because the zip checker checks the major version
   327  		// that's being published to, so the above path check also implicitly checks that.
   328  		return nil, nil, fmt.Errorf("major version %q found in %s does not match version being published %q", major, f.Name, m.Version())
   329  	}
   330  	// Check that all dependency versions look valid.
   331  	for modPath, dep := range mf.Deps {
   332  		_, err := module.NewVersion(modPath, dep.Version)
   333  		if err != nil {
   334  			return nil, nil, fmt.Errorf("invalid dependency: %v @ %v", modPath, dep.Version)
   335  		}
   336  	}
   337  	return data, mf, nil
   338  }
   339  
   340  // Module represents a CUE module instance.
   341  type Module struct {
   342  	client         *Client
   343  	loc            RegistryLocation
   344  	version        module.Version
   345  	manifest       ocispec.Manifest
   346  	manifestDigest ociregistry.Digest
   347  }
   348  
   349  func (m *Module) Version() module.Version {
   350  	return m.version
   351  }
   352  
   353  // ModuleFile returns the contents of the cue.mod/module.cue file.
   354  func (m *Module) ModuleFile(ctx context.Context) ([]byte, error) {
   355  	r, err := m.loc.Registry.GetBlob(ctx, m.loc.Repository, m.manifest.Layers[1].Digest)
   356  	if err != nil {
   357  		return nil, err
   358  	}
   359  	defer r.Close()
   360  	return io.ReadAll(r)
   361  }
   362  
   363  // GetZip returns a reader that can be used to read the contents of the zip
   364  // archive containing the module files. The reader should be closed after use,
   365  // and the contents should not be assumed to be correct until the close
   366  // error has been checked.
   367  func (m *Module) GetZip(ctx context.Context) (io.ReadCloser, error) {
   368  	return m.loc.Registry.GetBlob(ctx, m.loc.Repository, m.manifest.Layers[0].Digest)
   369  }
   370  
   371  // ManifestDigest returns the digest of the manifest representing
   372  // the module.
   373  func (m *Module) ManifestDigest() ociregistry.Digest {
   374  	return m.manifestDigest
   375  }
   376  
   377  func (c *Client) resolve(m module.Version) (RegistryLocation, error) {
   378  	loc, err := c.resolver.ResolveToRegistry(m.BasePath(), m.Version())
   379  	if err != nil {
   380  		return RegistryLocation{}, err
   381  	}
   382  	if loc.Registry == nil {
   383  		return RegistryLocation{}, fmt.Errorf("module %v unexpectedly resolved to nil registry", m)
   384  	}
   385  	if loc.Repository == "" {
   386  		return RegistryLocation{}, fmt.Errorf("module %v unexpectedly resolved to empty location", m)
   387  	}
   388  	if loc.Tag == "" {
   389  		return RegistryLocation{}, fmt.Errorf("module %v unexpectedly resolved to empty tag", m)
   390  	}
   391  	return loc, nil
   392  }
   393  
   394  func unmarshalManifest(ctx context.Context, data []byte, mediaType string) (*ociregistry.Manifest, error) {
   395  	if !isJSON(mediaType) {
   396  		return nil, fmt.Errorf("expected JSON media type but %q does not look like JSON", mediaType)
   397  	}
   398  	var m ociregistry.Manifest
   399  	if err := json.Unmarshal(data, &m); err != nil {
   400  		return nil, fmt.Errorf("cannot decode %s content as manifest: %v", mediaType, err)
   401  	}
   402  	return &m, nil
   403  }
   404  
   405  func isNotExist(err error) bool {
   406  	return errors.Is(err, ociregistry.ErrNameUnknown) || errors.Is(err, ociregistry.ErrNameInvalid)
   407  }
   408  
   409  func isModule(m *ocispec.Manifest) bool {
   410  	// TODO check m.ArtifactType too when that's defined?
   411  	// See https://github.com/opencontainers/image-spec/blob/main/manifest.md#image-manifest-property-descriptions
   412  	return m.Config.MediaType == moduleArtifactType
   413  }
   414  
   415  func isModuleFile(desc ocispec.Descriptor) bool {
   416  	return desc.ArtifactType == moduleFileMediaType ||
   417  		desc.MediaType == moduleFileMediaType
   418  }
   419  
   420  // isJSON reports whether the given media type has JSON as an underlying encoding.
   421  // TODO this is a guess. There's probably a more correct way to do it.
   422  func isJSON(mediaType string) bool {
   423  	return strings.HasSuffix(mediaType, "+json") || strings.HasSuffix(mediaType, "/json")
   424  }
   425  
   426  // scratchConfig returns a dummy configuration consisting only of the
   427  // two-byte configuration {}.
   428  // https://github.com/opencontainers/image-spec/blob/main/manifest.md#example-of-a-scratch-config-or-layer-descriptor
   429  func (c *Client) scratchConfig(ctx context.Context, loc RegistryLocation, mediaType string) (ocispec.Descriptor, error) {
   430  	// TODO check if it exists already to avoid push?
   431  	content := []byte("{}")
   432  	desc := ocispec.Descriptor{
   433  		Digest:    digest.FromBytes(content),
   434  		MediaType: mediaType,
   435  		Size:      int64(len(content)),
   436  	}
   437  	if _, err := loc.Registry.PushBlob(ctx, loc.Repository, desc, bytes.NewReader(content)); err != nil {
   438  		return ocispec.Descriptor{}, err
   439  	}
   440  	return desc, nil
   441  }
   442  
   443  // singleResolver implements Resolver by always returning R,
   444  // and mapping module paths directly to repository paths in
   445  // the registry.
   446  type singleResolver struct {
   447  	R ociregistry.Interface
   448  }
   449  
   450  func (r singleResolver) ResolveToRegistry(mpath, vers string) (RegistryLocation, error) {
   451  	return RegistryLocation{
   452  		Registry:   r.R,
   453  		Repository: mpath,
   454  		Tag:        vers,
   455  	}, nil
   456  }
   457
View as plain text