fetch.go

Documentation: cuelang.org/go/mod/modcache

     1  package modcache
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"io"
     8  	"io/fs"
     9  	"log"
    10  	"math/rand"
    11  	"os"
    12  	"path/filepath"
    13  	"strconv"
    14  	"strings"
    15  
    16  	"github.com/rogpeppe/go-internal/robustio"
    17  
    18  	"cuelang.org/go/internal/mod/modload"
    19  	"cuelang.org/go/internal/par"
    20  	"cuelang.org/go/mod/modfile"
    21  	"cuelang.org/go/mod/modregistry"
    22  	"cuelang.org/go/mod/module"
    23  	"cuelang.org/go/mod/modzip"
    24  )
    25  
    26  const logging = false // TODO hook this up to CUE_DEBUG
    27  
    28  // New returns r wrapped inside a caching layer that
    29  // stores persistent cached content inside the given
    30  // OS directory.
    31  //
    32  // The `module.SourceLoc.FS` fields in the locations
    33  // returned by the registry implement the `OSRootFS` interface,
    34  // allowing a caller to find the native OS filepath where modules
    35  // are stored.
    36  func New(registry *modregistry.Client, dir string) (modload.Registry, error) {
    37  	info, err := os.Stat(dir)
    38  	if err == nil && !info.IsDir() {
    39  		return nil, fmt.Errorf("%q is not a directory", dir)
    40  	}
    41  	return &cache{
    42  		dir: dir,
    43  		reg: registry,
    44  	}, nil
    45  }
    46  
    47  type cache struct {
    48  	dir              string
    49  	reg              *modregistry.Client
    50  	downloadZipCache par.ErrCache[module.Version, string]
    51  	modFileCache     par.ErrCache[string, []byte]
    52  }
    53  
    54  func (c *cache) Requirements(ctx context.Context, mv module.Version) ([]module.Version, error) {
    55  	data, err := c.downloadModFile(ctx, mv)
    56  	if err != nil {
    57  		return nil, err
    58  	}
    59  	mf, err := modfile.Parse(data, mv.String())
    60  	if err != nil {
    61  		return nil, fmt.Errorf("cannot parse module file from %v: %v", mv, err)
    62  	}
    63  	return mf.DepVersions(), nil
    64  }
    65  
    66  // Fetch returns the location of the contents for the given module
    67  // version, downloading it if necessary.
    68  func (c *cache) Fetch(ctx context.Context, mv module.Version) (module.SourceLoc, error) {
    69  	dir, err := c.downloadDir(ctx, mv)
    70  	if err == nil {
    71  		// The directory has already been completely extracted (no .partial file exists).
    72  		return c.dirToLocation(dir), nil
    73  	}
    74  	if dir == "" || !errors.Is(err, fs.ErrNotExist) {
    75  		return module.SourceLoc{}, err
    76  	}
    77  
    78  	// To avoid cluttering the cache with extraneous files,
    79  	// DownloadZip uses the same lockfile as Download.
    80  	// Invoke DownloadZip before locking the file.
    81  	zipfile, err := c.downloadZip(ctx, mv)
    82  	if err != nil {
    83  		return module.SourceLoc{}, err
    84  	}
    85  
    86  	unlock, err := c.lockVersion(ctx, mv)
    87  	if err != nil {
    88  		return module.SourceLoc{}, err
    89  	}
    90  	defer unlock()
    91  
    92  	// Check whether the directory was populated while we were waiting on the lock.
    93  	_, dirErr := c.downloadDir(ctx, mv)
    94  	if dirErr == nil {
    95  		return c.dirToLocation(dir), nil
    96  	}
    97  	_, dirExists := dirErr.(*downloadDirPartialError)
    98  
    99  	// Clean up any partially extracted directories (indicated by
   100  	// DownloadDirPartialError, usually because of a .partial file). This is only
   101  	// safe to do because the lock file ensures that their writers are no longer
   102  	// active.
   103  	parentDir := filepath.Dir(dir)
   104  	tmpPrefix := filepath.Base(dir) + ".tmp-"
   105  
   106  	entries, _ := os.ReadDir(parentDir)
   107  	for _, entry := range entries {
   108  		if strings.HasPrefix(entry.Name(), tmpPrefix) {
   109  			RemoveAll(filepath.Join(parentDir, entry.Name())) // best effort
   110  		}
   111  	}
   112  	if dirExists {
   113  		if err := RemoveAll(dir); err != nil {
   114  			return module.SourceLoc{}, err
   115  		}
   116  	}
   117  
   118  	partialPath, err := c.cachePath(ctx, mv, "partial")
   119  	if err != nil {
   120  		return module.SourceLoc{}, err
   121  	}
   122  
   123  	// Extract the module zip directory at its final location.
   124  	//
   125  	// To prevent other processes from reading the directory if we crash,
   126  	// create a .partial file before extracting the directory, and delete
   127  	// the .partial file afterward (all while holding the lock).
   128  	//
   129  	// A technique used previously was to extract to a temporary directory with a random name
   130  	// then rename it into place with os.Rename. On Windows, this can fail with
   131  	// ERROR_ACCESS_DENIED when another process (usually an anti-virus scanner)
   132  	// opened files in the temporary directory.
   133  	if err := os.MkdirAll(parentDir, 0777); err != nil {
   134  		return module.SourceLoc{}, err
   135  	}
   136  	if err := os.WriteFile(partialPath, nil, 0666); err != nil {
   137  		return module.SourceLoc{}, err
   138  	}
   139  	if err := modzip.Unzip(dir, mv, zipfile); err != nil {
   140  		if rmErr := RemoveAll(dir); rmErr == nil {
   141  			os.Remove(partialPath)
   142  		}
   143  		return module.SourceLoc{}, err
   144  	}
   145  	if err := os.Remove(partialPath); err != nil {
   146  		return module.SourceLoc{}, err
   147  	}
   148  	makeDirsReadOnly(dir)
   149  	return c.dirToLocation(dir), nil
   150  }
   151  
   152  // ModuleVersions implements [modload.Registry.ModuleVersions].
   153  func (c *cache) ModuleVersions(ctx context.Context, mpath string) ([]string, error) {
   154  	// TODO should this do any kind of short-term caching?
   155  	return c.reg.ModuleVersions(ctx, mpath)
   156  }
   157  
   158  func (c *cache) downloadZip(ctx context.Context, mv module.Version) (zipfile string, err error) {
   159  	return c.downloadZipCache.Do(mv, func() (string, error) {
   160  		zipfile, err := c.cachePath(ctx, mv, "zip")
   161  		if err != nil {
   162  			return "", err
   163  		}
   164  
   165  		// Return without locking if the zip file exists.
   166  		if _, err := os.Stat(zipfile); err == nil {
   167  			return zipfile, nil
   168  		}
   169  		logf("cue: downloading %s", mv)
   170  		unlock, err := c.lockVersion(ctx, mv)
   171  		if err != nil {
   172  			return "", err
   173  		}
   174  		defer unlock()
   175  
   176  		if err := c.downloadZip1(ctx, mv, zipfile); err != nil {
   177  			return "", err
   178  		}
   179  		return zipfile, nil
   180  	})
   181  }
   182  
   183  func (c *cache) downloadZip1(ctx context.Context, mod module.Version, zipfile string) (err error) {
   184  	// Double-check that the zipfile was not created while we were waiting for
   185  	// the lock in downloadZip.
   186  	if _, err := os.Stat(zipfile); err == nil {
   187  		return nil
   188  	}
   189  
   190  	// Create parent directories.
   191  	if err := os.MkdirAll(filepath.Dir(zipfile), 0777); err != nil {
   192  		return err
   193  	}
   194  
   195  	// Clean up any remaining tempfiles from previous runs.
   196  	// This is only safe to do because the lock file ensures that their
   197  	// writers are no longer active.
   198  	tmpPattern := filepath.Base(zipfile) + "*.tmp"
   199  	if old, err := filepath.Glob(filepath.Join(quoteGlob(filepath.Dir(zipfile)), tmpPattern)); err == nil {
   200  		for _, path := range old {
   201  			os.Remove(path) // best effort
   202  		}
   203  	}
   204  
   205  	// From here to the os.Rename call below is functionally almost equivalent to
   206  	// renameio.WriteToFile. We avoid using that so that we have control over the
   207  	// names of the temporary files (see the cleanup above) and to avoid adding
   208  	// renameio as an extra dependency.
   209  	f, err := tempFile(ctx, filepath.Dir(zipfile), filepath.Base(zipfile), 0666)
   210  	if err != nil {
   211  		return err
   212  	}
   213  	defer func() {
   214  		if err != nil {
   215  			f.Close()
   216  			os.Remove(f.Name())
   217  		}
   218  	}()
   219  
   220  	// TODO cache the result of GetModule so we don't have to do
   221  	// an extra round trip when we've already fetched the module file.
   222  	m, err := c.reg.GetModule(ctx, mod)
   223  	if err != nil {
   224  		return err
   225  	}
   226  	r, err := m.GetZip(ctx)
   227  	if err != nil {
   228  		return err
   229  	}
   230  	defer r.Close()
   231  	if _, err := io.Copy(f, r); err != nil {
   232  		return fmt.Errorf("failed to get module zip contents: %v", err)
   233  	}
   234  	if err := f.Close(); err != nil {
   235  		return err
   236  	}
   237  	if err := os.Rename(f.Name(), zipfile); err != nil {
   238  		return err
   239  	}
   240  	// TODO should we check the zip file for well-formedness?
   241  	// TODO: Should we make the .zip file read-only to discourage tampering?
   242  	return nil
   243  }
   244  
   245  func (c *cache) downloadModFile(ctx context.Context, mod module.Version) ([]byte, error) {
   246  	return c.modFileCache.Do(mod.String(), func() ([]byte, error) {
   247  		modfile, data, err := c.readDiskModFile(ctx, mod)
   248  		if err == nil {
   249  			return data, nil
   250  		}
   251  		logf("cue: downloading %s", mod)
   252  		unlock, err := c.lockVersion(ctx, mod)
   253  		if err != nil {
   254  			return nil, err
   255  		}
   256  		defer unlock()
   257  		// Double-check that the file hasn't been created while we were
   258  		// acquiring the lock.
   259  		_, data, err = c.readDiskModFile(ctx, mod)
   260  		if err == nil {
   261  			return data, nil
   262  		}
   263  		return c.downloadModFile1(ctx, mod, modfile)
   264  	})
   265  }
   266  
   267  func (c *cache) downloadModFile1(ctx context.Context, mod module.Version, modfile string) ([]byte, error) {
   268  	m, err := c.reg.GetModule(ctx, mod)
   269  	if err != nil {
   270  		return nil, err
   271  	}
   272  	data, err := m.ModuleFile(ctx)
   273  	if err != nil {
   274  		return nil, err
   275  	}
   276  	if err := c.writeDiskModFile(ctx, modfile, data); err != nil {
   277  		return nil, err
   278  	}
   279  	return data, nil
   280  }
   281  
   282  func (c *cache) dirToLocation(fpath string) module.SourceLoc {
   283  	return module.SourceLoc{
   284  		FS:  module.OSDirFS(fpath),
   285  		Dir: ".",
   286  	}
   287  }
   288  
   289  // makeDirsReadOnly makes a best-effort attempt to remove write permissions for dir
   290  // and its transitive contents.
   291  func makeDirsReadOnly(dir string) {
   292  	type pathMode struct {
   293  		path string
   294  		mode fs.FileMode
   295  	}
   296  	var dirs []pathMode // in lexical order
   297  	filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error {
   298  		if err == nil && d.IsDir() {
   299  			info, err := d.Info()
   300  			if err == nil && info.Mode()&0222 != 0 {
   301  				dirs = append(dirs, pathMode{path, info.Mode()})
   302  			}
   303  		}
   304  		return nil
   305  	})
   306  
   307  	// Run over list backward to chmod children before parents.
   308  	for i := len(dirs) - 1; i >= 0; i-- {
   309  		os.Chmod(dirs[i].path, dirs[i].mode&^0222)
   310  	}
   311  }
   312  
   313  // RemoveAll removes a directory written by the cache, first applying
   314  // any permission changes needed to do so.
   315  func RemoveAll(dir string) error {
   316  	// Module cache has 0555 directories; make them writable in order to remove content.
   317  	filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error {
   318  		if err != nil {
   319  			return nil // ignore errors walking in file system
   320  		}
   321  		if info.IsDir() {
   322  			os.Chmod(path, 0777)
   323  		}
   324  		return nil
   325  	})
   326  	return robustio.RemoveAll(dir)
   327  }
   328  
   329  // quoteGlob returns s with all Glob metacharacters quoted.
   330  // We don't try to handle backslash here, as that can appear in a
   331  // file path on Windows.
   332  func quoteGlob(s string) string {
   333  	if !strings.ContainsAny(s, `*?[]`) {
   334  		return s
   335  	}
   336  	var sb strings.Builder
   337  	for _, c := range s {
   338  		switch c {
   339  		case '*', '?', '[', ']':
   340  			sb.WriteByte('\\')
   341  		}
   342  		sb.WriteRune(c)
   343  	}
   344  	return sb.String()
   345  }
   346  
   347  // tempFile creates a new temporary file with given permission bits.
   348  func tempFile(ctx context.Context, dir, prefix string, perm fs.FileMode) (f *os.File, err error) {
   349  	for i := 0; i < 10000; i++ {
   350  		name := filepath.Join(dir, prefix+strconv.Itoa(rand.Intn(1000000000))+".tmp")
   351  		f, err = os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm)
   352  		if os.IsExist(err) {
   353  			if ctx.Err() != nil {
   354  				return nil, ctx.Err()
   355  			}
   356  			continue
   357  		}
   358  		break
   359  	}
   360  	return
   361  }
   362  
   363  func logf(f string, a ...any) {
   364  	if logging {
   365  		log.Printf(f, a...)
   366  	}
   367  }
   368
View as plain text