mod_cache.go

Documentation: golang.org/x/tools/internal/imports

     1  // Copyright 2019 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package imports
     6  
     7  import (
     8  	"context"
     9  	"fmt"
    10  	"path"
    11  	"path/filepath"
    12  	"strings"
    13  	"sync"
    14  
    15  	"golang.org/x/mod/module"
    16  	"golang.org/x/tools/internal/gopathwalk"
    17  	"golang.org/x/tools/internal/stdlib"
    18  )
    19  
    20  // To find packages to import, the resolver needs to know about all of
    21  // the packages that could be imported. This includes packages that are
    22  // already in modules that are in (1) the current module, (2) replace targets,
    23  // and (3) packages in the module cache. Packages in (1) and (2) may change over
    24  // time, as the client may edit the current module and locally replaced modules.
    25  // The module cache (which includes all of the packages in (3)) can only
    26  // ever be added to.
    27  //
    28  // The resolver can thus save state about packages in the module cache
    29  // and guarantee that this will not change over time. To obtain information
    30  // about new modules added to the module cache, the module cache should be
    31  // rescanned.
    32  //
    33  // It is OK to serve information about modules that have been deleted,
    34  // as they do still exist.
    35  // TODO(suzmue): can we share information with the caller about
    36  // what module needs to be downloaded to import this package?
    37  
    38  type directoryPackageStatus int
    39  
    40  const (
    41  	_ directoryPackageStatus = iota
    42  	directoryScanned
    43  	nameLoaded
    44  	exportsLoaded
    45  )
    46  
    47  // directoryPackageInfo holds (possibly incomplete) information about packages
    48  // contained in a given directory.
    49  type directoryPackageInfo struct {
    50  	// status indicates the extent to which this struct has been filled in.
    51  	status directoryPackageStatus
    52  	// err is non-nil when there was an error trying to reach status.
    53  	err error
    54  
    55  	// Set when status >= directoryScanned.
    56  
    57  	// dir is the absolute directory of this package.
    58  	dir      string
    59  	rootType gopathwalk.RootType
    60  	// nonCanonicalImportPath is the package's expected import path. It may
    61  	// not actually be importable at that path.
    62  	nonCanonicalImportPath string
    63  
    64  	// Module-related information.
    65  	moduleDir  string // The directory that is the module root of this dir.
    66  	moduleName string // The module name that contains this dir.
    67  
    68  	// Set when status >= nameLoaded.
    69  
    70  	packageName string // the package name, as declared in the source.
    71  
    72  	// Set when status >= exportsLoaded.
    73  	// TODO(rfindley): it's hard to see this, but exports depend implicitly on
    74  	// the default build context GOOS and GOARCH.
    75  	//
    76  	// We can make this explicit, and key exports by GOOS, GOARCH.
    77  	exports []stdlib.Symbol
    78  }
    79  
    80  // reachedStatus returns true when info has a status at least target and any error associated with
    81  // an attempt to reach target.
    82  func (info *directoryPackageInfo) reachedStatus(target directoryPackageStatus) (bool, error) {
    83  	if info.err == nil {
    84  		return info.status >= target, nil
    85  	}
    86  	if info.status == target {
    87  		return true, info.err
    88  	}
    89  	return true, nil
    90  }
    91  
    92  // DirInfoCache is a concurrency-safe map for storing information about
    93  // directories that may contain packages.
    94  //
    95  // The information in this cache is built incrementally. Entries are initialized in scan.
    96  // No new keys should be added in any other functions, as all directories containing
    97  // packages are identified in scan.
    98  //
    99  // Other functions, including loadExports and findPackage, may update entries in this cache
   100  // as they discover new things about the directory.
   101  //
   102  // The information in the cache is not expected to change for the cache's
   103  // lifetime, so there is no protection against competing writes. Users should
   104  // take care not to hold the cache across changes to the underlying files.
   105  type DirInfoCache struct {
   106  	mu sync.Mutex
   107  	// dirs stores information about packages in directories, keyed by absolute path.
   108  	dirs      map[string]*directoryPackageInfo
   109  	listeners map[*int]cacheListener
   110  }
   111  
   112  func NewDirInfoCache() *DirInfoCache {
   113  	return &DirInfoCache{
   114  		dirs:      make(map[string]*directoryPackageInfo),
   115  		listeners: make(map[*int]cacheListener),
   116  	}
   117  }
   118  
   119  type cacheListener func(directoryPackageInfo)
   120  
   121  // ScanAndListen calls listener on all the items in the cache, and on anything
   122  // newly added. The returned stop function waits for all in-flight callbacks to
   123  // finish and blocks new ones.
   124  func (d *DirInfoCache) ScanAndListen(ctx context.Context, listener cacheListener) func() {
   125  	ctx, cancel := context.WithCancel(ctx)
   126  
   127  	// Flushing out all the callbacks is tricky without knowing how many there
   128  	// are going to be. Setting an arbitrary limit makes it much easier.
   129  	const maxInFlight = 10
   130  	sema := make(chan struct{}, maxInFlight)
   131  	for i := 0; i < maxInFlight; i++ {
   132  		sema <- struct{}{}
   133  	}
   134  
   135  	cookie := new(int) // A unique ID we can use for the listener.
   136  
   137  	// We can't hold mu while calling the listener.
   138  	d.mu.Lock()
   139  	var keys []string
   140  	for key := range d.dirs {
   141  		keys = append(keys, key)
   142  	}
   143  	d.listeners[cookie] = func(info directoryPackageInfo) {
   144  		select {
   145  		case <-ctx.Done():
   146  			return
   147  		case <-sema:
   148  		}
   149  		listener(info)
   150  		sema <- struct{}{}
   151  	}
   152  	d.mu.Unlock()
   153  
   154  	stop := func() {
   155  		cancel()
   156  		d.mu.Lock()
   157  		delete(d.listeners, cookie)
   158  		d.mu.Unlock()
   159  		for i := 0; i < maxInFlight; i++ {
   160  			<-sema
   161  		}
   162  	}
   163  
   164  	// Process the pre-existing keys.
   165  	for _, k := range keys {
   166  		select {
   167  		case <-ctx.Done():
   168  			return stop
   169  		default:
   170  		}
   171  		if v, ok := d.Load(k); ok {
   172  			listener(v)
   173  		}
   174  	}
   175  
   176  	return stop
   177  }
   178  
   179  // Store stores the package info for dir.
   180  func (d *DirInfoCache) Store(dir string, info directoryPackageInfo) {
   181  	d.mu.Lock()
   182  	// TODO(rfindley, golang/go#59216): should we overwrite an existing entry?
   183  	// That seems incorrect as the cache should be idempotent.
   184  	_, old := d.dirs[dir]
   185  	d.dirs[dir] = &info
   186  	var listeners []cacheListener
   187  	for _, l := range d.listeners {
   188  		listeners = append(listeners, l)
   189  	}
   190  	d.mu.Unlock()
   191  
   192  	if !old {
   193  		for _, l := range listeners {
   194  			l(info)
   195  		}
   196  	}
   197  }
   198  
   199  // Load returns a copy of the directoryPackageInfo for absolute directory dir.
   200  func (d *DirInfoCache) Load(dir string) (directoryPackageInfo, bool) {
   201  	d.mu.Lock()
   202  	defer d.mu.Unlock()
   203  	info, ok := d.dirs[dir]
   204  	if !ok {
   205  		return directoryPackageInfo{}, false
   206  	}
   207  	return *info, true
   208  }
   209  
   210  // Keys returns the keys currently present in d.
   211  func (d *DirInfoCache) Keys() (keys []string) {
   212  	d.mu.Lock()
   213  	defer d.mu.Unlock()
   214  	for key := range d.dirs {
   215  		keys = append(keys, key)
   216  	}
   217  	return keys
   218  }
   219  
   220  func (d *DirInfoCache) CachePackageName(info directoryPackageInfo) (string, error) {
   221  	if loaded, err := info.reachedStatus(nameLoaded); loaded {
   222  		return info.packageName, err
   223  	}
   224  	if scanned, err := info.reachedStatus(directoryScanned); !scanned || err != nil {
   225  		return "", fmt.Errorf("cannot read package name, scan error: %v", err)
   226  	}
   227  	info.packageName, info.err = packageDirToName(info.dir)
   228  	info.status = nameLoaded
   229  	d.Store(info.dir, info)
   230  	return info.packageName, info.err
   231  }
   232  
   233  func (d *DirInfoCache) CacheExports(ctx context.Context, env *ProcessEnv, info directoryPackageInfo) (string, []stdlib.Symbol, error) {
   234  	if reached, _ := info.reachedStatus(exportsLoaded); reached {
   235  		return info.packageName, info.exports, info.err
   236  	}
   237  	if reached, err := info.reachedStatus(nameLoaded); reached && err != nil {
   238  		return "", nil, err
   239  	}
   240  	info.packageName, info.exports, info.err = loadExportsFromFiles(ctx, env, info.dir, false)
   241  	if info.err == context.Canceled || info.err == context.DeadlineExceeded {
   242  		return info.packageName, info.exports, info.err
   243  	}
   244  	// The cache structure wants things to proceed linearly. We can skip a
   245  	// step here, but only if we succeed.
   246  	if info.status == nameLoaded || info.err == nil {
   247  		info.status = exportsLoaded
   248  	} else {
   249  		info.status = nameLoaded
   250  	}
   251  	d.Store(info.dir, info)
   252  	return info.packageName, info.exports, info.err
   253  }
   254  
   255  // ScanModuleCache walks the given directory, which must be a GOMODCACHE value,
   256  // for directory package information, storing the results in cache.
   257  func ScanModuleCache(dir string, cache *DirInfoCache, logf func(string, ...any)) {
   258  	// Note(rfindley): it's hard to see, but this function attempts to implement
   259  	// just the side effects on cache of calling PrimeCache with a ProcessEnv
   260  	// that has the given dir as its GOMODCACHE.
   261  	//
   262  	// Teasing out the control flow, we see that we can avoid any handling of
   263  	// vendor/ and can infer module info entirely from the path, simplifying the
   264  	// logic here.
   265  
   266  	root := gopathwalk.Root{
   267  		Path: filepath.Clean(dir),
   268  		Type: gopathwalk.RootModuleCache,
   269  	}
   270  
   271  	directoryInfo := func(root gopathwalk.Root, dir string) directoryPackageInfo {
   272  		// This is a copy of ModuleResolver.scanDirForPackage, trimmed down to
   273  		// logic that applies to a module cache directory.
   274  
   275  		subdir := ""
   276  		if dir != root.Path {
   277  			subdir = dir[len(root.Path)+len("/"):]
   278  		}
   279  
   280  		matches := modCacheRegexp.FindStringSubmatch(subdir)
   281  		if len(matches) == 0 {
   282  			return directoryPackageInfo{
   283  				status: directoryScanned,
   284  				err:    fmt.Errorf("invalid module cache path: %v", subdir),
   285  			}
   286  		}
   287  		modPath, err := module.UnescapePath(filepath.ToSlash(matches[1]))
   288  		if err != nil {
   289  			if logf != nil {
   290  				logf("decoding module cache path %q: %v", subdir, err)
   291  			}
   292  			return directoryPackageInfo{
   293  				status: directoryScanned,
   294  				err:    fmt.Errorf("decoding module cache path %q: %v", subdir, err),
   295  			}
   296  		}
   297  		importPath := path.Join(modPath, filepath.ToSlash(matches[3]))
   298  		index := strings.Index(dir, matches[1]+"@"+matches[2])
   299  		modDir := filepath.Join(dir[:index], matches[1]+"@"+matches[2])
   300  		modName := readModName(filepath.Join(modDir, "go.mod"))
   301  		return directoryPackageInfo{
   302  			status:                 directoryScanned,
   303  			dir:                    dir,
   304  			rootType:               root.Type,
   305  			nonCanonicalImportPath: importPath,
   306  			moduleDir:              modDir,
   307  			moduleName:             modName,
   308  		}
   309  	}
   310  
   311  	add := func(root gopathwalk.Root, dir string) {
   312  		info := directoryInfo(root, dir)
   313  		cache.Store(info.dir, info)
   314  	}
   315  
   316  	skip := func(_ gopathwalk.Root, dir string) bool {
   317  		// Skip directories that have already been scanned.
   318  		//
   319  		// Note that gopathwalk only adds "package" directories, which must contain
   320  		// a .go file, and all such package directories in the module cache are
   321  		// immutable. So if we can load a dir, it can be skipped.
   322  		info, ok := cache.Load(dir)
   323  		if !ok {
   324  			return false
   325  		}
   326  		packageScanned, _ := info.reachedStatus(directoryScanned)
   327  		return packageScanned
   328  	}
   329  
   330  	gopathwalk.WalkSkip([]gopathwalk.Root{root}, add, skip, gopathwalk.Options{Logf: logf, ModulesEnabled: true})
   331  }
   332
View as plain text